{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "967acdf8-b5f2-4746-a56f-28fdfc27595d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============GPU================\n",
      "Fri Dec 22 00:10:53 2023       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 4090        On  | 00000000:25:00.0 Off |                  Off |\n",
      "| 30%   24C    P8              22W / 450W |      2MiB / 24564MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA GeForce RTX 4090        On  | 00000000:41:00.0 Off |                  Off |\n",
      "| 30%   22C    P8              19W / 450W |      2MiB / 24564MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|  No running processes found                                                           |\n",
      "+---------------------------------------------------------------------------------------+\n",
      "============CUDA version================\n",
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2023 NVIDIA Corporation\n",
      "Built on Mon_Apr__3_17:16:06_PDT_2023\n",
      "Cuda compilation tools, release 12.1, V12.1.105\n",
      "Build cuda_12.1.r12.1/compiler.32688072_0\n",
      "============CPU================\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "model name\t: AMD EPYC 7B13 64-Core Processor\n",
      "============Memory================\n",
      "MemTotal:       1056617192 kB\n"
     ]
    }
   ],
   "source": [
    "# GPU\n",
    "print(\"============GPU================\")\n",
    "!nvidia-smi\n",
    "\n",
    "# CUDA version\n",
    "print(\"============CUDA version================\")\n",
    "!nvcc --version\n",
    "\n",
    "# CPU\n",
    "print(\"============CPU================\")\n",
    "!cat /proc/cpuinfo | grep model\\ name\n",
    "\n",
    "# Memory\n",
    "print(\"============Memory================\")\n",
    "!cat /proc/meminfo | grep MemTotal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bb0c71b2-b7d2-47b2-82ab-24619929a13d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llama'...\n",
      "remote: Enumerating objects: 417, done.\u001b[K\n",
      "remote: Counting objects: 100% (71/71), done.\u001b[K\n",
      "remote: Compressing objects: 100% (49/49), done.\u001b[K\n",
      "remote: Total 417 (delta 29), reused 48 (delta 14), pack-reused 346\u001b[K\n",
      "Receiving objects: 100% (417/417), 1.10 MiB | 10.41 MiB/s, done.\n",
      "Resolving deltas: 100% (214/214), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/facebookresearch/llama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "080f46e7-5783-4fc1-9552-59f9021bdfc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "%cd llama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e764773d-63c6-4076-bc0b-79583e7927b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Downloading LICENSE and Acceptable Usage Policy\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/LICENSE?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.87, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n",
      "\n",
      "    The file is already fully retrieved; nothing to do.\n",
      "\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/USE_POLICY.md?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n",
      "\n",
      "    The file is already fully retrieved; nothing to do.\n",
      "\n",
      "Downloading tokenizer\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/tokenizer.model?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 499723 (488K) [binary/octet-stream]\n",
      "Saving to: ‘./tokenizer.model’\n",
      "\n",
      "./tokenizer.model   100%[===================>] 488.01K  --.-KB/s    in 0.07s   \n",
      "\n",
      "2023-12-21 18:46:25 (7.11 MB/s) - ‘./tokenizer.model’ saved [499723/499723]\n",
      "\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/tokenizer_checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.87, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 50 [binary/octet-stream]\n",
      "Saving to: ‘./tokenizer_checklist.chk’\n",
      "\n",
      "./tokenizer_checkli 100%[===================>]      50  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:46:26 (69.1 MB/s) - ‘./tokenizer_checklist.chk’ saved [50/50]\n",
      "\n",
      "tokenizer.model: OK\n",
      "Downloading llama-2-7b\n",
      "--2023-12-21 18:46:26--  https://download.llamameta.net/llama-2-7b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13476925163 (13G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-7b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-7b/consol 100%[===================>]  12.55G  35.9MB/s    in 5m 13s  \n",
      "\n",
      "2023-12-21 18:51:39 (41.1 MB/s) - ‘./llama-2-7b/consolidated.00.pth’ saved [13476925163/13476925163]\n",
      "\n",
      "--2023-12-21 18:51:39--  https://download.llamameta.net/llama-2-7b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 102 [application/json]\n",
      "Saving to: ‘./llama-2-7b/params.json’\n",
      "\n",
      "./llama-2-7b/params 100%[===================>]     102  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:51:39 (10.1 MB/s) - ‘./llama-2-7b/params.json’ saved [102/102]\n",
      "\n",
      "--2023-12-21 18:51:39--  https://download.llamameta.net/llama-2-7b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 100 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-7b/checklist.chk’\n",
      "\n",
      "./llama-2-7b/checkl 100%[===================>]     100  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:51:39 (119 MB/s) - ‘./llama-2-7b/checklist.chk’ saved [100/100]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "params.json: OK\n",
      "Downloading llama-2-13b\n",
      "--2023-12-21 18:52:04--  https://download.llamameta.net/llama-2-13b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13016329643 (12G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-13b/conso 100%[===================>]  12.12G  60.8MB/s    in 3m 59s  \n",
      "\n",
      "2023-12-21 18:56:03 (52.0 MB/s) - ‘./llama-2-13b/consolidated.00.pth’ saved [13016329643/13016329643]\n",
      "\n",
      "--2023-12-21 18:56:03--  https://download.llamameta.net/llama-2-13b/consolidated.01.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13016329643 (12G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/consolidated.01.pth’\n",
      "\n",
      "./llama-2-13b/conso 100%[===================>]  12.12G  71.7MB/s    in 4m 1s   \n",
      "\n",
      "2023-12-21 19:00:05 (51.6 MB/s) - ‘./llama-2-13b/consolidated.01.pth’ saved [13016329643/13016329643]\n",
      "\n",
      "--2023-12-21 19:00:05--  https://download.llamameta.net/llama-2-13b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 102 [application/json]\n",
      "Saving to: ‘./llama-2-13b/params.json’\n",
      "\n",
      "./llama-2-13b/param 100%[===================>]     102  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:00:05 (12.3 MB/s) - ‘./llama-2-13b/params.json’ saved [102/102]\n",
      "\n",
      "--2023-12-21 19:00:05--  https://download.llamameta.net/llama-2-13b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 154 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/checklist.chk’\n",
      "\n",
      "./llama-2-13b/check 100%[===================>]     154  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:00:05 (229 MB/s) - ‘./llama-2-13b/checklist.chk’ saved [154/154]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "consolidated.01.pth: OK\n",
      "params.json: OK\n",
      "Downloading llama-2-70b\n",
      "--2023-12-21 19:00:53--  https://download.llamameta.net/llama-2-70b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  99.3MB/s    in 3m 12s  \n",
      "\n",
      "2023-12-21 19:04:05 (85.8 MB/s) - ‘./llama-2-70b/consolidated.00.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:04:05--  https://download.llamameta.net/llama-2-70b/consolidated.01.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.01.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  73.7MB/s    in 3m 28s  \n",
      "\n",
      "2023-12-21 19:07:33 (79.1 MB/s) - ‘./llama-2-70b/consolidated.01.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:07:33--  https://download.llamameta.net/llama-2-70b/consolidated.02.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.02.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  97.8MB/s    in 3m 21s  \n",
      "\n",
      "2023-12-21 19:10:55 (81.7 MB/s) - ‘./llama-2-70b/consolidated.02.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:10:55--  https://download.llamameta.net/llama-2-70b/consolidated.03.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.87, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.03.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  88.7MB/s    in 3m 39s  \n",
      "\n",
      "2023-12-21 19:14:34 (75.0 MB/s) - ‘./llama-2-70b/consolidated.03.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:14:34--  https://download.llamameta.net/llama-2-70b/consolidated.04.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.04.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  9.11MB/s    in 3m 59s  \n",
      "\n",
      "2023-12-21 19:18:34 (68.8 MB/s) - ‘./llama-2-70b/consolidated.04.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:18:34--  https://download.llamameta.net/llama-2-70b/consolidated.05.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.05.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  91.5MB/s    in 3m 9s   \n",
      "\n",
      "2023-12-21 19:21:42 (87.2 MB/s) - ‘./llama-2-70b/consolidated.05.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:21:42--  https://download.llamameta.net/llama-2-70b/consolidated.06.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.06.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  88.5MB/s    in 3m 0s   \n",
      "\n",
      "2023-12-21 19:24:43 (91.3 MB/s) - ‘./llama-2-70b/consolidated.06.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:24:43--  https://download.llamameta.net/llama-2-70b/consolidated.07.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.07.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  46.3MB/s    in 4m 20s  \n",
      "\n",
      "2023-12-21 19:29:03 (63.2 MB/s) - ‘./llama-2-70b/consolidated.07.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:29:03--  https://download.llamameta.net/llama-2-70b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 147 [application/json]\n",
      "Saving to: ‘./llama-2-70b/params.json’\n",
      "\n",
      "./llama-2-70b/param 100%[===================>]     147  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:29:04 (1.68 MB/s) - ‘./llama-2-70b/params.json’ saved [147/147]\n",
      "\n",
      "--2023-12-21 19:29:04--  https://download.llamameta.net/llama-2-70b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 478 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/checklist.chk’\n",
      "\n",
      "./llama-2-70b/check 100%[===================>]     478  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:29:04 (38.1 MB/s) - ‘./llama-2-70b/checklist.chk’ saved [478/478]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "consolidated.01.pth: OK\n",
      "consolidated.02.pth: OK\n",
      "consolidated.03.pth: OK\n",
      "consolidated.04.pth: OK\n",
      "consolidated.05.pth: OK\n",
      "consolidated.06.pth: OK\n",
      "consolidated.07.pth: OK\n",
      "params.json: OK\n"
     ]
    }
   ],
   "source": [
    "# Define your PRESIGNED_URL and MODEL_SIZE in the script to prevent asking in the notebook\n",
    "!bash download.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1541dcc9-b822-4783-b6eb-020fc4a0316d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace\n"
     ]
    }
   ],
   "source": [
    "%cd /workspace\n",
    "!mkdir -p llama.cpp/models/7B-v2/\n",
    "!mv llama/llama-2-7b/* llama.cpp/models/7B-v2/\n",
    "!mkdir -p llama.cpp/models/13B-v2/\n",
    "!mv llama/llama-2-13b/* llama.cpp/models/13B-v2/\n",
    "!mkdir -p llama.cpp/models/70B-v2/\n",
    "!mv llama/llama-2-70b/* llama.cpp/models/70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cea2bab8-7c0d-42cc-8e32-064e71a58a74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "%cd llama.cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9594719a-ea2a-4b8d-bd26-2c19f0a6a2de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 13283  100 13283    0     0  73817      0 --:--:-- --:--:-- --:--:-- 73794\n"
     ]
    }
   ],
   "source": [
    "# If you encounter the error \"does not appear to have a file named config.json\" when converting the models to ggml FP16 format, try to convert the model to huggingface format to get the config.json file.\n",
    "!curl -o convert_llama_weights_to_hf.py https://raw.githubusercontent.com/huggingface/transformers/main/src/transformers/models/llama/convert_llama_weights_to_hf.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06b44ccf-4bf6-47a4-8f05-87a662822110",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp/models\n"
     ]
    }
   ],
   "source": [
    "%cd models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8f541028-baaa-40f8-8e1c-c359b5ead34c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp tokenizer.model 7B-v2/\n",
    "!cp tokenizer.model 13B-v2/\n",
    "!cp tokenizer.model 70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fcc944e7-32a8-4918-8001-6b51fb835377",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ae14bc36-f1ba-4069-82bc-63242471a393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting numpy==1.24.4 (from -r requirements.txt (line 1))\n",
      "  Using cached numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB)\n",
      "Collecting sentencepiece==0.1.98 (from -r requirements.txt (line 2))\n",
      "  Using cached sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Collecting transformers>=4.34.0 (from -r requirements.txt (line 3))\n",
      "  Using cached transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n",
      "Collecting gguf>=0.1.0 (from -r requirements.txt (line 4))\n",
      "  Using cached gguf-0.6.0-py3-none-any.whl.metadata (3.2 kB)\n",
      "Collecting protobuf>=4.21.0 (from -r requirements.txt (line 5))\n",
      "  Using cached protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl.metadata (541 bytes)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (3.13.1)\n",
      "Collecting huggingface-hub<1.0,>=0.19.3 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached huggingface_hub-0.20.1-py3-none-any.whl.metadata (12 kB)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (6.0.1)\n",
      "Collecting regex!=2019.12.17 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (2.31.0)\n",
      "Collecting tokenizers<0.19,>=0.14 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
      "Collecting safetensors>=0.3.1 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached safetensors-0.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
      "Collecting tqdm>=4.27 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached tqdm-4.66.1-py3-none-any.whl.metadata (57 kB)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers>=4.34.0->-r requirements.txt (line 3)) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers>=4.34.0->-r requirements.txt (line 3)) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (2023.11.17)\n",
      "Using cached numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "Using cached transformers-4.36.2-py3-none-any.whl (8.2 MB)\n",
      "Using cached gguf-0.6.0-py3-none-any.whl (23 kB)\n",
      "Using cached protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl (294 kB)\n",
      "Using cached huggingface_hub-0.20.1-py3-none-any.whl (330 kB)\n",
      "Using cached regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (773 kB)\n",
      "Using cached safetensors-0.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Using cached tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
      "Using cached tqdm-4.66.1-py3-none-any.whl (78 kB)\n",
      "Installing collected packages: sentencepiece, tqdm, safetensors, regex, protobuf, numpy, huggingface-hub, gguf, tokenizers, transformers\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.26.2\n",
      "    Uninstalling numpy-1.26.2:\n",
      "      Successfully uninstalled numpy-1.26.2\n",
      "Successfully installed gguf-0.6.0 huggingface-hub-0.20.1 numpy-1.24.4 protobuf-4.25.1 regex-2023.10.3 safetensors-0.4.1 sentencepiece-0.1.98 tokenizers-0.15.0 tqdm-4.66.1 transformers-4.36.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# !pip uninstall accelerate # If you have this package, uninstall it first, then use `convert to hf model` to get the config.json.\n",
    "# install Python dependencies\n",
    "!python3 -m pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a95b3a38-17fb-4792-b1cd-4255e6ed2a7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/7B-v2/.\n",
      "Loading the checkpoint in a Llama model.\n",
      "Traceback (most recent call last):\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 319, in <module>\n",
      "    main()\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 307, in main\n",
      "    write_model(\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 271, in write_model\n",
      "    model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 2863, in from_pretrained\n",
      "    raise ImportError(\n",
      "ImportError: Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`\n",
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/13B-v2/.\n",
      "Loading the checkpoint in a Llama model.\n",
      "Traceback (most recent call last):\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 319, in <module>\n",
      "    main()\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 307, in main\n",
      "    write_model(\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 271, in write_model\n",
      "    model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 2863, in from_pretrained\n",
      "    raise ImportError(\n",
      "ImportError: Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`\n",
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/70B-v2/.\n"
     ]
    }
   ],
   "source": [
    "# We don't need these models actually. We only need this to figure out the config.json error.\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/7B-v2/ --model_size 7B --output_dir models/7B-v2/\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/13B-v2/ --model_size 13B --output_dir models/13B-v2/\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/70B-v2/ --model_size 70B --output_dir models/70B-v2/ # Surprisingly, it still solves the problem although you can't find the config.json file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "839375fa-44f5-498c-8f1e-0e22ad8311ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Edit your params.json file if the \"vocab_size\" mismatch\n",
    "import json\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/7B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/7B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/13B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/13B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/70B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/70B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c5ce9c63-6f03-4736-a1df-56b9605f698b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model file models/7B-v2/consolidated.00.pth\n",
      "params = Params(n_vocab=32000, n_embd=4096, n_layer=32, n_ctx=4096, n_ff=11008, n_head=32, n_head_kv=32, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/7B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 4096]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [4096]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 4096]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [4096]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/7B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/291] Writing tensor token_embd.weight                      | size  32000 x   4096  | type F16  | T+   1\n",
      "[  2/291] Writing tensor output_norm.weight                     | size   4096           | type F32  | T+   1\n",
      "[  3/291] Writing tensor output.weight                          | size  32000 x   4096  | type F16  | T+   1\n",
      "[  4/291] Writing tensor blk.0.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  5/291] Writing tensor blk.0.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  6/291] Writing tensor blk.0.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  7/291] Writing tensor blk.0.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
      "[  8/291] Writing tensor blk.0.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
      "[  9/291] Writing tensor blk.0.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
      "[ 10/291] Writing tensor blk.0.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 11/291] Writing tensor blk.0.attn_norm.weight                 | size   4096           | type F32  | T+   1\n",
      "[ 12/291] Writing tensor blk.0.ffn_norm.weight                  | size   4096           | type F32  | T+   1\n",
      "[ 13/291] Writing tensor blk.1.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 14/291] Writing tensor blk.1.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 15/291] Writing tensor blk.1.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 16/291] Writing tensor blk.1.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 17/291] Writing tensor blk.1.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 18/291] Writing tensor blk.1.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
      "[ 19/291] Writing tensor blk.1.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 20/291] Writing tensor blk.1.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 21/291] Writing tensor blk.1.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 22/291] Writing tensor blk.2.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 23/291] Writing tensor blk.2.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 24/291] Writing tensor blk.2.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 25/291] Writing tensor blk.2.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 26/291] Writing tensor blk.2.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 27/291] Writing tensor blk.2.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
      "[ 28/291] Writing tensor blk.2.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 29/291] Writing tensor blk.2.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 30/291] Writing tensor blk.2.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 31/291] Writing tensor blk.3.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 32/291] Writing tensor blk.3.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 33/291] Writing tensor blk.3.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 34/291] Writing tensor blk.3.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 35/291] Writing tensor blk.3.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 36/291] Writing tensor blk.3.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
      "[ 37/291] Writing tensor blk.3.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 38/291] Writing tensor blk.3.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 39/291] Writing tensor blk.3.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 40/291] Writing tensor blk.4.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 41/291] Writing tensor blk.4.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 42/291] Writing tensor blk.4.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 43/291] Writing tensor blk.4.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 44/291] Writing tensor blk.4.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 45/291] Writing tensor blk.4.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 46/291] Writing tensor blk.4.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 47/291] Writing tensor blk.4.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
      "[ 48/291] Writing tensor blk.4.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
      "[ 49/291] Writing tensor blk.5.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 50/291] Writing tensor blk.5.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 51/291] Writing tensor blk.5.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 52/291] Writing tensor blk.5.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 53/291] Writing tensor blk.5.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 54/291] Writing tensor blk.5.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 55/291] Writing tensor blk.5.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 56/291] Writing tensor blk.5.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
      "[ 57/291] Writing tensor blk.5.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
      "[ 58/291] Writing tensor blk.6.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 59/291] Writing tensor blk.6.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 60/291] Writing tensor blk.6.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 61/291] Writing tensor blk.6.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 62/291] Writing tensor blk.6.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 63/291] Writing tensor blk.6.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 64/291] Writing tensor blk.6.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 65/291] Writing tensor blk.6.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 66/291] Writing tensor blk.6.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 67/291] Writing tensor blk.7.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 68/291] Writing tensor blk.7.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 69/291] Writing tensor blk.7.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 70/291] Writing tensor blk.7.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 71/291] Writing tensor blk.7.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 72/291] Writing tensor blk.7.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
      "[ 73/291] Writing tensor blk.7.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 74/291] Writing tensor blk.7.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 75/291] Writing tensor blk.7.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 76/291] Writing tensor blk.8.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 77/291] Writing tensor blk.8.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 78/291] Writing tensor blk.8.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 79/291] Writing tensor blk.8.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 80/291] Writing tensor blk.8.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 81/291] Writing tensor blk.8.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
      "[ 82/291] Writing tensor blk.8.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 83/291] Writing tensor blk.8.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 84/291] Writing tensor blk.8.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 85/291] Writing tensor blk.9.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 86/291] Writing tensor blk.9.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 87/291] Writing tensor blk.9.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 88/291] Writing tensor blk.9.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 89/291] Writing tensor blk.9.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 90/291] Writing tensor blk.9.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   5\n",
      "[ 91/291] Writing tensor blk.9.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 92/291] Writing tensor blk.9.attn_norm.weight                 | size   4096           | type F32  | T+   5\n",
      "[ 93/291] Writing tensor blk.9.ffn_norm.weight                  | size   4096           | type F32  | T+   5\n",
      "[ 94/291] Writing tensor blk.10.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 95/291] Writing tensor blk.10.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 96/291] Writing tensor blk.10.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 97/291] Writing tensor blk.10.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 98/291] Writing tensor blk.10.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 99/291] Writing tensor blk.10.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   5\n",
      "[100/291] Writing tensor blk.10.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   5\n",
      "[101/291] Writing tensor blk.10.attn_norm.weight                | size   4096           | type F32  | T+   5\n",
      "[102/291] Writing tensor blk.10.ffn_norm.weight                 | size   4096           | type F32  | T+   5\n",
      "[103/291] Writing tensor blk.11.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[104/291] Writing tensor blk.11.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[105/291] Writing tensor blk.11.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[106/291] Writing tensor blk.11.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
      "[107/291] Writing tensor blk.11.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
      "[108/291] Writing tensor blk.11.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[109/291] Writing tensor blk.11.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[110/291] Writing tensor blk.11.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[111/291] Writing tensor blk.11.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[112/291] Writing tensor blk.12.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[113/291] Writing tensor blk.12.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[114/291] Writing tensor blk.12.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[115/291] Writing tensor blk.12.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[116/291] Writing tensor blk.12.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   6\n",
      "[117/291] Writing tensor blk.12.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[118/291] Writing tensor blk.12.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[119/291] Writing tensor blk.12.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[120/291] Writing tensor blk.12.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[121/291] Writing tensor blk.13.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[122/291] Writing tensor blk.13.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[123/291] Writing tensor blk.13.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[124/291] Writing tensor blk.13.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[125/291] Writing tensor blk.13.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   6\n",
      "[126/291] Writing tensor blk.13.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[127/291] Writing tensor blk.13.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[128/291] Writing tensor blk.13.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[129/291] Writing tensor blk.13.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[130/291] Writing tensor blk.14.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[131/291] Writing tensor blk.14.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[132/291] Writing tensor blk.14.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[133/291] Writing tensor blk.14.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[134/291] Writing tensor blk.14.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[135/291] Writing tensor blk.14.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   7\n",
      "[136/291] Writing tensor blk.14.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   7\n",
      "[137/291] Writing tensor blk.14.attn_norm.weight                | size   4096           | type F32  | T+   7\n",
      "[138/291] Writing tensor blk.14.ffn_norm.weight                 | size   4096           | type F32  | T+   7\n",
      "[139/291] Writing tensor blk.15.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[140/291] Writing tensor blk.15.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[141/291] Writing tensor blk.15.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[142/291] Writing tensor blk.15.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
      "[143/291] Writing tensor blk.15.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[144/291] Writing tensor blk.15.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   7\n",
      "[145/291] Writing tensor blk.15.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   7\n",
      "[146/291] Writing tensor blk.15.attn_norm.weight                | size   4096           | type F32  | T+   7\n",
      "[147/291] Writing tensor blk.15.ffn_norm.weight                 | size   4096           | type F32  | T+   7\n",
      "[148/291] Writing tensor blk.16.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[149/291] Writing tensor blk.16.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[150/291] Writing tensor blk.16.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[151/291] Writing tensor blk.16.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
      "[152/291] Writing tensor blk.16.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[153/291] Writing tensor blk.16.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[154/291] Writing tensor blk.16.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[155/291] Writing tensor blk.16.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[156/291] Writing tensor blk.16.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[157/291] Writing tensor blk.17.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[158/291] Writing tensor blk.17.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[159/291] Writing tensor blk.17.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[160/291] Writing tensor blk.17.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[161/291] Writing tensor blk.17.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   8\n",
      "[162/291] Writing tensor blk.17.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[163/291] Writing tensor blk.17.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[164/291] Writing tensor blk.17.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[165/291] Writing tensor blk.17.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[166/291] Writing tensor blk.18.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[167/291] Writing tensor blk.18.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[168/291] Writing tensor blk.18.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[169/291] Writing tensor blk.18.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[170/291] Writing tensor blk.18.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   8\n",
      "[171/291] Writing tensor blk.18.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[172/291] Writing tensor blk.18.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[173/291] Writing tensor blk.18.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[174/291] Writing tensor blk.18.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[175/291] Writing tensor blk.19.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[176/291] Writing tensor blk.19.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[177/291] Writing tensor blk.19.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[178/291] Writing tensor blk.19.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[179/291] Writing tensor blk.19.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[180/291] Writing tensor blk.19.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
      "[181/291] Writing tensor blk.19.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   9\n",
      "[182/291] Writing tensor blk.19.attn_norm.weight                | size   4096           | type F32  | T+   9\n",
      "[183/291] Writing tensor blk.19.ffn_norm.weight                 | size   4096           | type F32  | T+   9\n",
      "[184/291] Writing tensor blk.20.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[185/291] Writing tensor blk.20.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[186/291] Writing tensor blk.20.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[187/291] Writing tensor blk.20.attn_output.weight              | size   4096 x   4096  | type F16  | T+   9\n",
      "[188/291] Writing tensor blk.20.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[189/291] Writing tensor blk.20.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
      "[190/291] Writing tensor blk.20.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   9\n",
      "[191/291] Writing tensor blk.20.attn_norm.weight                | size   4096           | type F32  | T+   9\n",
      "[192/291] Writing tensor blk.20.ffn_norm.weight                 | size   4096           | type F32  | T+   9\n",
      "[193/291] Writing tensor blk.21.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[194/291] Writing tensor blk.21.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[195/291] Writing tensor blk.21.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[196/291] Writing tensor blk.21.attn_output.weight              | size   4096 x   4096  | type F16  | T+   9\n",
      "[197/291] Writing tensor blk.21.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[198/291] Writing tensor blk.21.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[199/291] Writing tensor blk.21.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[200/291] Writing tensor blk.21.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[201/291] Writing tensor blk.21.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[202/291] Writing tensor blk.22.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[203/291] Writing tensor blk.22.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[204/291] Writing tensor blk.22.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[205/291] Writing tensor blk.22.attn_output.weight              | size   4096 x   4096  | type F16  | T+  10\n",
      "[206/291] Writing tensor blk.22.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  10\n",
      "[207/291] Writing tensor blk.22.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[208/291] Writing tensor blk.22.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[209/291] Writing tensor blk.22.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[210/291] Writing tensor blk.22.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[211/291] Writing tensor blk.23.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[212/291] Writing tensor blk.23.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[213/291] Writing tensor blk.23.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[214/291] Writing tensor blk.23.attn_output.weight              | size   4096 x   4096  | type F16  | T+  10\n",
      "[215/291] Writing tensor blk.23.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  10\n",
      "[216/291] Writing tensor blk.23.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[217/291] Writing tensor blk.23.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[218/291] Writing tensor blk.23.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[219/291] Writing tensor blk.23.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[220/291] Writing tensor blk.24.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[221/291] Writing tensor blk.24.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[222/291] Writing tensor blk.24.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[223/291] Writing tensor blk.24.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[224/291] Writing tensor blk.24.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  11\n",
      "[225/291] Writing tensor blk.24.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
      "[226/291] Writing tensor blk.24.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
      "[227/291] Writing tensor blk.24.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
      "[228/291] Writing tensor blk.24.ffn_norm.weight                 | size   4096           | type F32  | T+  11\n",
      "[229/291] Writing tensor blk.25.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[230/291] Writing tensor blk.25.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[231/291] Writing tensor blk.25.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[232/291] Writing tensor blk.25.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[233/291] Writing tensor blk.25.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  11\n",
      "[234/291] Writing tensor blk.25.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
      "[235/291] Writing tensor blk.25.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
      "[236/291] Writing tensor blk.25.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
      "[237/291] Writing tensor blk.25.ffn_norm.weight                 | size   4096           | type F32  | T+  11\n",
      "[238/291] Writing tensor blk.26.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[239/291] Writing tensor blk.26.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[240/291] Writing tensor blk.26.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[241/291] Writing tensor blk.26.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[242/291] Writing tensor blk.26.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[243/291] Writing tensor blk.26.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[244/291] Writing tensor blk.26.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[245/291] Writing tensor blk.26.attn_norm.weight                | size   4096           | type F32  | T+  12\n",
      "[246/291] Writing tensor blk.26.ffn_norm.weight                 | size   4096           | type F32  | T+  12\n",
      "[247/291] Writing tensor blk.27.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[248/291] Writing tensor blk.27.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[249/291] Writing tensor blk.27.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[250/291] Writing tensor blk.27.attn_output.weight              | size   4096 x   4096  | type F16  | T+  12\n",
      "[251/291] Writing tensor blk.27.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[252/291] Writing tensor blk.27.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[253/291] Writing tensor blk.27.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[254/291] Writing tensor blk.27.attn_norm.weight                | size   4096           | type F32  | T+  12\n",
      "[255/291] Writing tensor blk.27.ffn_norm.weight                 | size   4096           | type F32  | T+  12\n",
      "[256/291] Writing tensor blk.28.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[257/291] Writing tensor blk.28.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[258/291] Writing tensor blk.28.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[259/291] Writing tensor blk.28.attn_output.weight              | size   4096 x   4096  | type F16  | T+  12\n",
      "[260/291] Writing tensor blk.28.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[261/291] Writing tensor blk.28.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[262/291] Writing tensor blk.28.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[263/291] Writing tensor blk.28.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[264/291] Writing tensor blk.28.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[265/291] Writing tensor blk.29.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[266/291] Writing tensor blk.29.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[267/291] Writing tensor blk.29.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[268/291] Writing tensor blk.29.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[269/291] Writing tensor blk.29.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
      "[270/291] Writing tensor blk.29.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  13\n",
      "[271/291] Writing tensor blk.29.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  13\n",
      "[272/291] Writing tensor blk.29.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[273/291] Writing tensor blk.29.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[274/291] Writing tensor blk.30.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[275/291] Writing tensor blk.30.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[276/291] Writing tensor blk.30.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[277/291] Writing tensor blk.30.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[278/291] Writing tensor blk.30.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
      "[279/291] Writing tensor blk.30.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  13\n",
      "[280/291] Writing tensor blk.30.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  13\n",
      "[281/291] Writing tensor blk.30.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[282/291] Writing tensor blk.30.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[283/291] Writing tensor blk.31.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[284/291] Writing tensor blk.31.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[285/291] Writing tensor blk.31.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[286/291] Writing tensor blk.31.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[287/291] Writing tensor blk.31.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  14\n",
      "[288/291] Writing tensor blk.31.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  14\n",
      "[289/291] Writing tensor blk.31.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  14\n",
      "[290/291] Writing tensor blk.31.attn_norm.weight                | size   4096           | type F32  | T+  14\n",
      "[291/291] Writing tensor blk.31.ffn_norm.weight                 | size   4096           | type F32  | T+  14\n",
      "Wrote models/7B-v2/ggml-model-f16.gguf\n",
      "Loading model file models/13B-v2/consolidated.00.pth\n",
      "Loading model file models/13B-v2/consolidated.01.pth\n",
      "params = Params(n_vocab=32000, n_embd=5120, n_layer=40, n_ctx=4096, n_ff=13824, n_head=40, n_head_kv=40, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/13B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 5120]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [5120]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 5120]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.32.attention.wq.weight                    -> blk.32.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wk.weight                    -> blk.32.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wv.weight                    -> blk.32.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wo.weight                    -> blk.32.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.32.feed_forward.w1.weight                 -> blk.32.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.32.feed_forward.w2.weight                 -> blk.32.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.32.feed_forward.w3.weight                 -> blk.32.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.32.attention_norm.weight                  -> blk.32.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.32.ffn_norm.weight                        -> blk.32.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.33.attention.wq.weight                    -> blk.33.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wk.weight                    -> blk.33.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wv.weight                    -> blk.33.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wo.weight                    -> blk.33.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.33.feed_forward.w1.weight                 -> blk.33.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.33.feed_forward.w2.weight                 -> blk.33.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.33.feed_forward.w3.weight                 -> blk.33.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.33.attention_norm.weight                  -> blk.33.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.33.ffn_norm.weight                        -> blk.33.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.34.attention.wq.weight                    -> blk.34.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wk.weight                    -> blk.34.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wv.weight                    -> blk.34.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wo.weight                    -> blk.34.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.34.feed_forward.w1.weight                 -> blk.34.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.34.feed_forward.w2.weight                 -> blk.34.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.34.feed_forward.w3.weight                 -> blk.34.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.34.attention_norm.weight                  -> blk.34.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.34.ffn_norm.weight                        -> blk.34.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.35.attention.wq.weight                    -> blk.35.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wk.weight                    -> blk.35.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wv.weight                    -> blk.35.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wo.weight                    -> blk.35.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.35.feed_forward.w1.weight                 -> blk.35.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.35.feed_forward.w2.weight                 -> blk.35.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.35.feed_forward.w3.weight                 -> blk.35.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.35.attention_norm.weight                  -> blk.35.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.35.ffn_norm.weight                        -> blk.35.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.36.attention.wq.weight                    -> blk.36.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wk.weight                    -> blk.36.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wv.weight                    -> blk.36.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wo.weight                    -> blk.36.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.36.feed_forward.w1.weight                 -> blk.36.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.36.feed_forward.w2.weight                 -> blk.36.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.36.feed_forward.w3.weight                 -> blk.36.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.36.attention_norm.weight                  -> blk.36.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.36.ffn_norm.weight                        -> blk.36.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.37.attention.wq.weight                    -> blk.37.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wk.weight                    -> blk.37.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wv.weight                    -> blk.37.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wo.weight                    -> blk.37.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.37.feed_forward.w1.weight                 -> blk.37.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.37.feed_forward.w2.weight                 -> blk.37.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.37.feed_forward.w3.weight                 -> blk.37.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.37.attention_norm.weight                  -> blk.37.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.37.ffn_norm.weight                        -> blk.37.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.38.attention.wq.weight                    -> blk.38.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wk.weight                    -> blk.38.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wv.weight                    -> blk.38.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wo.weight                    -> blk.38.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.38.feed_forward.w1.weight                 -> blk.38.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.38.feed_forward.w2.weight                 -> blk.38.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.38.feed_forward.w3.weight                 -> blk.38.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.38.attention_norm.weight                  -> blk.38.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.38.ffn_norm.weight                        -> blk.38.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.39.attention.wq.weight                    -> blk.39.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wk.weight                    -> blk.39.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wv.weight                    -> blk.39.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wo.weight                    -> blk.39.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.39.feed_forward.w1.weight                 -> blk.39.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.39.feed_forward.w2.weight                 -> blk.39.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.39.feed_forward.w3.weight                 -> blk.39.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.39.attention_norm.weight                  -> blk.39.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.39.ffn_norm.weight                        -> blk.39.ffn_norm.weight                   | BF16   | [5120]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/13B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/363] Writing tensor token_embd.weight                      | size  32000 x   5120  | type F16  | T+   1\n",
      "[  2/363] Writing tensor output_norm.weight                     | size   5120           | type F32  | T+   1\n",
      "[  3/363] Writing tensor output.weight                          | size  32000 x   5120  | type F16  | T+   1\n",
      "[  4/363] Writing tensor blk.0.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  5/363] Writing tensor blk.0.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  6/363] Writing tensor blk.0.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  7/363] Writing tensor blk.0.attn_output.weight               | size   5120 x   5120  | type F16  | T+   1\n",
      "[  8/363] Writing tensor blk.0.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   1\n",
      "[  9/363] Writing tensor blk.0.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   2\n",
      "[ 10/363] Writing tensor blk.0.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   2\n",
      "[ 11/363] Writing tensor blk.0.attn_norm.weight                 | size   5120           | type F32  | T+   2\n",
      "[ 12/363] Writing tensor blk.0.ffn_norm.weight                  | size   5120           | type F32  | T+   2\n",
      "[ 13/363] Writing tensor blk.1.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 14/363] Writing tensor blk.1.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 15/363] Writing tensor blk.1.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 16/363] Writing tensor blk.1.attn_output.weight               | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 17/363] Writing tensor blk.1.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   2\n",
      "[ 18/363] Writing tensor blk.1.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   3\n",
      "[ 19/363] Writing tensor blk.1.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   3\n",
      "[ 20/363] Writing tensor blk.1.attn_norm.weight                 | size   5120           | type F32  | T+   3\n",
      "[ 21/363] Writing tensor blk.1.ffn_norm.weight                  | size   5120           | type F32  | T+   3\n",
      "[ 22/363] Writing tensor blk.2.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 23/363] Writing tensor blk.2.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 24/363] Writing tensor blk.2.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 25/363] Writing tensor blk.2.attn_output.weight               | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 26/363] Writing tensor blk.2.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   4\n",
      "[ 27/363] Writing tensor blk.2.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   4\n",
      "[ 28/363] Writing tensor blk.2.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   5\n",
      "[ 29/363] Writing tensor blk.2.attn_norm.weight                 | size   5120           | type F32  | T+   5\n",
      "[ 30/363] Writing tensor blk.2.ffn_norm.weight                  | size   5120           | type F32  | T+   5\n",
      "[ 31/363] Writing tensor blk.3.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 32/363] Writing tensor blk.3.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 33/363] Writing tensor blk.3.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 34/363] Writing tensor blk.3.attn_output.weight               | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 35/363] Writing tensor blk.3.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   5\n",
      "[ 36/363] Writing tensor blk.3.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   6\n",
      "[ 37/363] Writing tensor blk.3.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   6\n",
      "[ 38/363] Writing tensor blk.3.attn_norm.weight                 | size   5120           | type F32  | T+   6\n",
      "[ 39/363] Writing tensor blk.3.ffn_norm.weight                  | size   5120           | type F32  | T+   6\n",
      "[ 40/363] Writing tensor blk.4.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 41/363] Writing tensor blk.4.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 42/363] Writing tensor blk.4.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 43/363] Writing tensor blk.4.attn_output.weight               | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 44/363] Writing tensor blk.4.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   6\n",
      "[ 45/363] Writing tensor blk.4.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   7\n",
      "[ 46/363] Writing tensor blk.4.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   7\n",
      "[ 47/363] Writing tensor blk.4.attn_norm.weight                 | size   5120           | type F32  | T+   7\n",
      "[ 48/363] Writing tensor blk.4.ffn_norm.weight                  | size   5120           | type F32  | T+   7\n",
      "[ 49/363] Writing tensor blk.5.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 50/363] Writing tensor blk.5.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 51/363] Writing tensor blk.5.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 52/363] Writing tensor blk.5.attn_output.weight               | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 53/363] Writing tensor blk.5.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   8\n",
      "[ 54/363] Writing tensor blk.5.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   8\n",
      "[ 55/363] Writing tensor blk.5.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   8\n",
      "[ 56/363] Writing tensor blk.5.attn_norm.weight                 | size   5120           | type F32  | T+   8\n",
      "[ 57/363] Writing tensor blk.5.ffn_norm.weight                  | size   5120           | type F32  | T+   8\n",
      "[ 58/363] Writing tensor blk.6.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 59/363] Writing tensor blk.6.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 60/363] Writing tensor blk.6.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 61/363] Writing tensor blk.6.attn_output.weight               | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 62/363] Writing tensor blk.6.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   9\n",
      "[ 63/363] Writing tensor blk.6.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   9\n",
      "[ 64/363] Writing tensor blk.6.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   9\n",
      "[ 65/363] Writing tensor blk.6.attn_norm.weight                 | size   5120           | type F32  | T+   9\n",
      "[ 66/363] Writing tensor blk.6.ffn_norm.weight                  | size   5120           | type F32  | T+   9\n",
      "[ 67/363] Writing tensor blk.7.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   9\n",
      "[ 68/363] Writing tensor blk.7.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 69/363] Writing tensor blk.7.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 70/363] Writing tensor blk.7.attn_output.weight               | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 71/363] Writing tensor blk.7.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  10\n",
      "[ 72/363] Writing tensor blk.7.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  10\n",
      "[ 73/363] Writing tensor blk.7.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  11\n",
      "[ 74/363] Writing tensor blk.7.attn_norm.weight                 | size   5120           | type F32  | T+  11\n",
      "[ 75/363] Writing tensor blk.7.ffn_norm.weight                  | size   5120           | type F32  | T+  11\n",
      "[ 76/363] Writing tensor blk.8.attn_q.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 77/363] Writing tensor blk.8.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 78/363] Writing tensor blk.8.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 79/363] Writing tensor blk.8.attn_output.weight               | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 80/363] Writing tensor blk.8.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  11\n",
      "[ 81/363] Writing tensor blk.8.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  12\n",
      "[ 82/363] Writing tensor blk.8.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  12\n",
      "[ 83/363] Writing tensor blk.8.attn_norm.weight                 | size   5120           | type F32  | T+  12\n",
      "[ 84/363] Writing tensor blk.8.ffn_norm.weight                  | size   5120           | type F32  | T+  12\n",
      "[ 85/363] Writing tensor blk.9.attn_q.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 86/363] Writing tensor blk.9.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 87/363] Writing tensor blk.9.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 88/363] Writing tensor blk.9.attn_output.weight               | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 89/363] Writing tensor blk.9.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  12\n",
      "[ 90/363] Writing tensor blk.9.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  13\n",
      "[ 91/363] Writing tensor blk.9.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  13\n",
      "[ 92/363] Writing tensor blk.9.attn_norm.weight                 | size   5120           | type F32  | T+  13\n",
      "[ 93/363] Writing tensor blk.9.ffn_norm.weight                  | size   5120           | type F32  | T+  13\n",
      "[ 94/363] Writing tensor blk.10.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 95/363] Writing tensor blk.10.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 96/363] Writing tensor blk.10.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 97/363] Writing tensor blk.10.attn_output.weight              | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 98/363] Writing tensor blk.10.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  14\n",
      "[ 99/363] Writing tensor blk.10.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  14\n",
      "[100/363] Writing tensor blk.10.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  14\n",
      "[101/363] Writing tensor blk.10.attn_norm.weight                | size   5120           | type F32  | T+  14\n",
      "[102/363] Writing tensor blk.10.ffn_norm.weight                 | size   5120           | type F32  | T+  14\n",
      "[103/363] Writing tensor blk.11.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[104/363] Writing tensor blk.11.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[105/363] Writing tensor blk.11.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[106/363] Writing tensor blk.11.attn_output.weight              | size   5120 x   5120  | type F16  | T+  14\n",
      "[107/363] Writing tensor blk.11.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  15\n",
      "[108/363] Writing tensor blk.11.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  15\n",
      "[109/363] Writing tensor blk.11.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  16\n",
      "[110/363] Writing tensor blk.11.attn_norm.weight                | size   5120           | type F32  | T+  16\n",
      "[111/363] Writing tensor blk.11.ffn_norm.weight                 | size   5120           | type F32  | T+  16\n",
      "[112/363] Writing tensor blk.12.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[113/363] Writing tensor blk.12.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[114/363] Writing tensor blk.12.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[115/363] Writing tensor blk.12.attn_output.weight              | size   5120 x   5120  | type F16  | T+  16\n",
      "[116/363] Writing tensor blk.12.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  16\n",
      "[117/363] Writing tensor blk.12.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  17\n",
      "[118/363] Writing tensor blk.12.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  17\n",
      "[119/363] Writing tensor blk.12.attn_norm.weight                | size   5120           | type F32  | T+  17\n",
      "[120/363] Writing tensor blk.12.ffn_norm.weight                 | size   5120           | type F32  | T+  17\n",
      "[121/363] Writing tensor blk.13.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[122/363] Writing tensor blk.13.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[123/363] Writing tensor blk.13.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[124/363] Writing tensor blk.13.attn_output.weight              | size   5120 x   5120  | type F16  | T+  17\n",
      "[125/363] Writing tensor blk.13.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  17\n",
      "[126/363] Writing tensor blk.13.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  18\n",
      "[127/363] Writing tensor blk.13.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  18\n",
      "[128/363] Writing tensor blk.13.attn_norm.weight                | size   5120           | type F32  | T+  18\n",
      "[129/363] Writing tensor blk.13.ffn_norm.weight                 | size   5120           | type F32  | T+  18\n",
      "[130/363] Writing tensor blk.14.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[131/363] Writing tensor blk.14.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[132/363] Writing tensor blk.14.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[133/363] Writing tensor blk.14.attn_output.weight              | size   5120 x   5120  | type F16  | T+  18\n",
      "[134/363] Writing tensor blk.14.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  19\n",
      "[135/363] Writing tensor blk.14.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  19\n",
      "[136/363] Writing tensor blk.14.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  19\n",
      "[137/363] Writing tensor blk.14.attn_norm.weight                | size   5120           | type F32  | T+  19\n",
      "[138/363] Writing tensor blk.14.ffn_norm.weight                 | size   5120           | type F32  | T+  19\n",
      "[139/363] Writing tensor blk.15.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[140/363] Writing tensor blk.15.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[141/363] Writing tensor blk.15.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[142/363] Writing tensor blk.15.attn_output.weight              | size   5120 x   5120  | type F16  | T+  19\n",
      "[143/363] Writing tensor blk.15.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  20\n",
      "[144/363] Writing tensor blk.15.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  20\n",
      "[145/363] Writing tensor blk.15.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  21\n",
      "[146/363] Writing tensor blk.15.attn_norm.weight                | size   5120           | type F32  | T+  21\n",
      "[147/363] Writing tensor blk.15.ffn_norm.weight                 | size   5120           | type F32  | T+  21\n",
      "[148/363] Writing tensor blk.16.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[149/363] Writing tensor blk.16.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[150/363] Writing tensor blk.16.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[151/363] Writing tensor blk.16.attn_output.weight              | size   5120 x   5120  | type F16  | T+  21\n",
      "[152/363] Writing tensor blk.16.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  21\n",
      "[153/363] Writing tensor blk.16.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  22\n",
      "[154/363] Writing tensor blk.16.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  22\n",
      "[155/363] Writing tensor blk.16.attn_norm.weight                | size   5120           | type F32  | T+  22\n",
      "[156/363] Writing tensor blk.16.ffn_norm.weight                 | size   5120           | type F32  | T+  22\n",
      "[157/363] Writing tensor blk.17.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[158/363] Writing tensor blk.17.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[159/363] Writing tensor blk.17.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[160/363] Writing tensor blk.17.attn_output.weight              | size   5120 x   5120  | type F16  | T+  22\n",
      "[161/363] Writing tensor blk.17.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  22\n",
      "[162/363] Writing tensor blk.17.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  23\n",
      "[163/363] Writing tensor blk.17.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  23\n",
      "[164/363] Writing tensor blk.17.attn_norm.weight                | size   5120           | type F32  | T+  23\n",
      "[165/363] Writing tensor blk.17.ffn_norm.weight                 | size   5120           | type F32  | T+  23\n",
      "[166/363] Writing tensor blk.18.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[167/363] Writing tensor blk.18.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[168/363] Writing tensor blk.18.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[169/363] Writing tensor blk.18.attn_output.weight              | size   5120 x   5120  | type F16  | T+  23\n",
      "[170/363] Writing tensor blk.18.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  24\n",
      "[171/363] Writing tensor blk.18.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  24\n",
      "[172/363] Writing tensor blk.18.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  24\n",
      "[173/363] Writing tensor blk.18.attn_norm.weight                | size   5120           | type F32  | T+  24\n",
      "[174/363] Writing tensor blk.18.ffn_norm.weight                 | size   5120           | type F32  | T+  24\n",
      "[175/363] Writing tensor blk.19.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[176/363] Writing tensor blk.19.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[177/363] Writing tensor blk.19.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[178/363] Writing tensor blk.19.attn_output.weight              | size   5120 x   5120  | type F16  | T+  24\n",
      "[179/363] Writing tensor blk.19.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  25\n",
      "[180/363] Writing tensor blk.19.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  25\n",
      "[181/363] Writing tensor blk.19.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  26\n",
      "[182/363] Writing tensor blk.19.attn_norm.weight                | size   5120           | type F32  | T+  26\n",
      "[183/363] Writing tensor blk.19.ffn_norm.weight                 | size   5120           | type F32  | T+  26\n",
      "[184/363] Writing tensor blk.20.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[185/363] Writing tensor blk.20.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[186/363] Writing tensor blk.20.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[187/363] Writing tensor blk.20.attn_output.weight              | size   5120 x   5120  | type F16  | T+  26\n",
      "[188/363] Writing tensor blk.20.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  26\n",
      "[189/363] Writing tensor blk.20.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  27\n",
      "[190/363] Writing tensor blk.20.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  27\n",
      "[191/363] Writing tensor blk.20.attn_norm.weight                | size   5120           | type F32  | T+  27\n",
      "[192/363] Writing tensor blk.20.ffn_norm.weight                 | size   5120           | type F32  | T+  27\n",
      "[193/363] Writing tensor blk.21.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[194/363] Writing tensor blk.21.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[195/363] Writing tensor blk.21.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[196/363] Writing tensor blk.21.attn_output.weight              | size   5120 x   5120  | type F16  | T+  27\n",
      "[197/363] Writing tensor blk.21.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  27\n",
      "[198/363] Writing tensor blk.21.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  28\n",
      "[199/363] Writing tensor blk.21.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  28\n",
      "[200/363] Writing tensor blk.21.attn_norm.weight                | size   5120           | type F32  | T+  28\n",
      "[201/363] Writing tensor blk.21.ffn_norm.weight                 | size   5120           | type F32  | T+  28\n",
      "[202/363] Writing tensor blk.22.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[203/363] Writing tensor blk.22.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[204/363] Writing tensor blk.22.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[205/363] Writing tensor blk.22.attn_output.weight              | size   5120 x   5120  | type F16  | T+  28\n",
      "[206/363] Writing tensor blk.22.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  29\n",
      "[207/363] Writing tensor blk.22.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  29\n",
      "[208/363] Writing tensor blk.22.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  29\n",
      "[209/363] Writing tensor blk.22.attn_norm.weight                | size   5120           | type F32  | T+  29\n",
      "[210/363] Writing tensor blk.22.ffn_norm.weight                 | size   5120           | type F32  | T+  29\n",
      "[211/363] Writing tensor blk.23.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[212/363] Writing tensor blk.23.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[213/363] Writing tensor blk.23.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[214/363] Writing tensor blk.23.attn_output.weight              | size   5120 x   5120  | type F16  | T+  29\n",
      "[215/363] Writing tensor blk.23.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  30\n",
      "[216/363] Writing tensor blk.23.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  30\n",
      "[217/363] Writing tensor blk.23.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  30\n",
      "[218/363] Writing tensor blk.23.attn_norm.weight                | size   5120           | type F32  | T+  30\n",
      "[219/363] Writing tensor blk.23.ffn_norm.weight                 | size   5120           | type F32  | T+  30\n",
      "[220/363] Writing tensor blk.24.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  30\n",
      "[221/363] Writing tensor blk.24.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  31\n",
      "[222/363] Writing tensor blk.24.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  31\n",
      "[223/363] Writing tensor blk.24.attn_output.weight              | size   5120 x   5120  | type F16  | T+  31\n",
      "[224/363] Writing tensor blk.24.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  31\n",
      "[225/363] Writing tensor blk.24.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  31\n",
      "[226/363] Writing tensor blk.24.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  32\n",
      "[227/363] Writing tensor blk.24.attn_norm.weight                | size   5120           | type F32  | T+  32\n",
      "[228/363] Writing tensor blk.24.ffn_norm.weight                 | size   5120           | type F32  | T+  32\n",
      "[229/363] Writing tensor blk.25.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[230/363] Writing tensor blk.25.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[231/363] Writing tensor blk.25.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[232/363] Writing tensor blk.25.attn_output.weight              | size   5120 x   5120  | type F16  | T+  32\n",
      "[233/363] Writing tensor blk.25.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  32\n",
      "[234/363] Writing tensor blk.25.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  33\n",
      "[235/363] Writing tensor blk.25.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  33\n",
      "[236/363] Writing tensor blk.25.attn_norm.weight                | size   5120           | type F32  | T+  33\n",
      "[237/363] Writing tensor blk.25.ffn_norm.weight                 | size   5120           | type F32  | T+  33\n",
      "[238/363] Writing tensor blk.26.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[239/363] Writing tensor blk.26.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[240/363] Writing tensor blk.26.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[241/363] Writing tensor blk.26.attn_output.weight              | size   5120 x   5120  | type F16  | T+  33\n",
      "[242/363] Writing tensor blk.26.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  33\n",
      "[243/363] Writing tensor blk.26.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  34\n",
      "[244/363] Writing tensor blk.26.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  34\n",
      "[245/363] Writing tensor blk.26.attn_norm.weight                | size   5120           | type F32  | T+  34\n",
      "[246/363] Writing tensor blk.26.ffn_norm.weight                 | size   5120           | type F32  | T+  34\n",
      "[247/363] Writing tensor blk.27.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[248/363] Writing tensor blk.27.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[249/363] Writing tensor blk.27.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[250/363] Writing tensor blk.27.attn_output.weight              | size   5120 x   5120  | type F16  | T+  34\n",
      "[251/363] Writing tensor blk.27.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  35\n",
      "[252/363] Writing tensor blk.27.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  35\n",
      "[253/363] Writing tensor blk.27.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  35\n",
      "[254/363] Writing tensor blk.27.attn_norm.weight                | size   5120           | type F32  | T+  35\n",
      "[255/363] Writing tensor blk.27.ffn_norm.weight                 | size   5120           | type F32  | T+  35\n",
      "[256/363] Writing tensor blk.28.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[257/363] Writing tensor blk.28.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[258/363] Writing tensor blk.28.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[259/363] Writing tensor blk.28.attn_output.weight              | size   5120 x   5120  | type F16  | T+  35\n",
      "[260/363] Writing tensor blk.28.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  36\n",
      "[261/363] Writing tensor blk.28.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  36\n",
      "[262/363] Writing tensor blk.28.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  36\n",
      "[263/363] Writing tensor blk.28.attn_norm.weight                | size   5120           | type F32  | T+  37\n",
      "[264/363] Writing tensor blk.28.ffn_norm.weight                 | size   5120           | type F32  | T+  37\n",
      "[265/363] Writing tensor blk.29.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[266/363] Writing tensor blk.29.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[267/363] Writing tensor blk.29.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[268/363] Writing tensor blk.29.attn_output.weight              | size   5120 x   5120  | type F16  | T+  37\n",
      "[269/363] Writing tensor blk.29.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  37\n",
      "[270/363] Writing tensor blk.29.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  37\n",
      "[271/363] Writing tensor blk.29.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  38\n",
      "[272/363] Writing tensor blk.29.attn_norm.weight                | size   5120           | type F32  | T+  38\n",
      "[273/363] Writing tensor blk.29.ffn_norm.weight                 | size   5120           | type F32  | T+  38\n",
      "[274/363] Writing tensor blk.30.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[275/363] Writing tensor blk.30.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[276/363] Writing tensor blk.30.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[277/363] Writing tensor blk.30.attn_output.weight              | size   5120 x   5120  | type F16  | T+  38\n",
      "[278/363] Writing tensor blk.30.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  38\n",
      "[279/363] Writing tensor blk.30.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  39\n",
      "[280/363] Writing tensor blk.30.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  39\n",
      "[281/363] Writing tensor blk.30.attn_norm.weight                | size   5120           | type F32  | T+  39\n",
      "[282/363] Writing tensor blk.30.ffn_norm.weight                 | size   5120           | type F32  | T+  39\n",
      "[283/363] Writing tensor blk.31.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[284/363] Writing tensor blk.31.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[285/363] Writing tensor blk.31.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[286/363] Writing tensor blk.31.attn_output.weight              | size   5120 x   5120  | type F16  | T+  39\n",
      "[287/363] Writing tensor blk.31.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  39\n",
      "[288/363] Writing tensor blk.31.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  40\n",
      "[289/363] Writing tensor blk.31.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  40\n",
      "[290/363] Writing tensor blk.31.attn_norm.weight                | size   5120           | type F32  | T+  40\n",
      "[291/363] Writing tensor blk.31.ffn_norm.weight                 | size   5120           | type F32  | T+  40\n",
      "[292/363] Writing tensor blk.32.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[293/363] Writing tensor blk.32.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[294/363] Writing tensor blk.32.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[295/363] Writing tensor blk.32.attn_output.weight              | size   5120 x   5120  | type F16  | T+  40\n",
      "[296/363] Writing tensor blk.32.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  41\n",
      "[297/363] Writing tensor blk.32.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  41\n",
      "[298/363] Writing tensor blk.32.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  41\n",
      "[299/363] Writing tensor blk.32.attn_norm.weight                | size   5120           | type F32  | T+  41\n",
      "[300/363] Writing tensor blk.32.ffn_norm.weight                 | size   5120           | type F32  | T+  41\n",
      "[301/363] Writing tensor blk.33.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[302/363] Writing tensor blk.33.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[303/363] Writing tensor blk.33.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[304/363] Writing tensor blk.33.attn_output.weight              | size   5120 x   5120  | type F16  | T+  41\n",
      "[305/363] Writing tensor blk.33.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  42\n",
      "[306/363] Writing tensor blk.33.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  42\n",
      "[307/363] Writing tensor blk.33.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  43\n",
      "[308/363] Writing tensor blk.33.attn_norm.weight                | size   5120           | type F32  | T+  43\n",
      "[309/363] Writing tensor blk.33.ffn_norm.weight                 | size   5120           | type F32  | T+  43\n",
      "[310/363] Writing tensor blk.34.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[311/363] Writing tensor blk.34.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[312/363] Writing tensor blk.34.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[313/363] Writing tensor blk.34.attn_output.weight              | size   5120 x   5120  | type F16  | T+  43\n",
      "[314/363] Writing tensor blk.34.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  43\n",
      "[315/363] Writing tensor blk.34.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  43\n",
      "[316/363] Writing tensor blk.34.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  44\n",
      "[317/363] Writing tensor blk.34.attn_norm.weight                | size   5120           | type F32  | T+  44\n",
      "[318/363] Writing tensor blk.34.ffn_norm.weight                 | size   5120           | type F32  | T+  44\n",
      "[319/363] Writing tensor blk.35.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[320/363] Writing tensor blk.35.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[321/363] Writing tensor blk.35.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[322/363] Writing tensor blk.35.attn_output.weight              | size   5120 x   5120  | type F16  | T+  44\n",
      "[323/363] Writing tensor blk.35.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  44\n",
      "[324/363] Writing tensor blk.35.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  45\n",
      "[325/363] Writing tensor blk.35.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  45\n",
      "[326/363] Writing tensor blk.35.attn_norm.weight                | size   5120           | type F32  | T+  45\n",
      "[327/363] Writing tensor blk.35.ffn_norm.weight                 | size   5120           | type F32  | T+  45\n",
      "[328/363] Writing tensor blk.36.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[329/363] Writing tensor blk.36.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[330/363] Writing tensor blk.36.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[331/363] Writing tensor blk.36.attn_output.weight              | size   5120 x   5120  | type F16  | T+  45\n",
      "[332/363] Writing tensor blk.36.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  45\n",
      "[333/363] Writing tensor blk.36.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  46\n",
      "[334/363] Writing tensor blk.36.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  46\n",
      "[335/363] Writing tensor blk.36.attn_norm.weight                | size   5120           | type F32  | T+  46\n",
      "[336/363] Writing tensor blk.36.ffn_norm.weight                 | size   5120           | type F32  | T+  46\n",
      "[337/363] Writing tensor blk.37.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[338/363] Writing tensor blk.37.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[339/363] Writing tensor blk.37.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[340/363] Writing tensor blk.37.attn_output.weight              | size   5120 x   5120  | type F16  | T+  46\n",
      "[341/363] Writing tensor blk.37.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  47\n",
      "[342/363] Writing tensor blk.37.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  47\n",
      "[343/363] Writing tensor blk.37.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  47\n",
      "[344/363] Writing tensor blk.37.attn_norm.weight                | size   5120           | type F32  | T+  47\n",
      "[345/363] Writing tensor blk.37.ffn_norm.weight                 | size   5120           | type F32  | T+  47\n",
      "[346/363] Writing tensor blk.38.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  47\n",
      "[347/363] Writing tensor blk.38.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  47\n",
      "[348/363] Writing tensor blk.38.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  48\n",
      "[349/363] Writing tensor blk.38.attn_output.weight              | size   5120 x   5120  | type F16  | T+  48\n",
      "[350/363] Writing tensor blk.38.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  48\n",
      "[351/363] Writing tensor blk.38.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  48\n",
      "[352/363] Writing tensor blk.38.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  49\n",
      "[353/363] Writing tensor blk.38.attn_norm.weight                | size   5120           | type F32  | T+  49\n",
      "[354/363] Writing tensor blk.38.ffn_norm.weight                 | size   5120           | type F32  | T+  49\n",
      "[355/363] Writing tensor blk.39.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[356/363] Writing tensor blk.39.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[357/363] Writing tensor blk.39.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[358/363] Writing tensor blk.39.attn_output.weight              | size   5120 x   5120  | type F16  | T+  49\n",
      "[359/363] Writing tensor blk.39.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  49\n",
      "[360/363] Writing tensor blk.39.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  50\n",
      "[361/363] Writing tensor blk.39.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  50\n",
      "[362/363] Writing tensor blk.39.attn_norm.weight                | size   5120           | type F32  | T+  50\n",
      "[363/363] Writing tensor blk.39.ffn_norm.weight                 | size   5120           | type F32  | T+  50\n",
      "Wrote models/13B-v2/ggml-model-f16.gguf\n",
      "Loading model file models/70B-v2/consolidated.00.pth\n",
      "Loading model file models/70B-v2/consolidated.01.pth\n",
      "Loading model file models/70B-v2/consolidated.02.pth\n",
      "Loading model file models/70B-v2/consolidated.03.pth\n",
      "Loading model file models/70B-v2/consolidated.04.pth\n",
      "Loading model file models/70B-v2/consolidated.05.pth\n",
      "Loading model file models/70B-v2/consolidated.06.pth\n",
      "Loading model file models/70B-v2/consolidated.07.pth\n",
      "params = Params(n_vocab=32000, n_embd=8192, n_layer=80, n_ctx=4096, n_ff=28672, n_head=64, n_head_kv=8, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/70B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 8192]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [8192]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 8192]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.32.attention.wq.weight                    -> blk.32.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.32.attention.wk.weight                    -> blk.32.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.32.attention.wv.weight                    -> blk.32.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.32.attention.wo.weight                    -> blk.32.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.32.feed_forward.w1.weight                 -> blk.32.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.32.feed_forward.w2.weight                 -> blk.32.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.32.feed_forward.w3.weight                 -> blk.32.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.32.attention_norm.weight                  -> blk.32.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.32.ffn_norm.weight                        -> blk.32.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.33.attention.wq.weight                    -> blk.33.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.33.attention.wk.weight                    -> blk.33.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.33.attention.wv.weight                    -> blk.33.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.33.attention.wo.weight                    -> blk.33.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.33.feed_forward.w1.weight                 -> blk.33.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.33.feed_forward.w2.weight                 -> blk.33.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.33.feed_forward.w3.weight                 -> blk.33.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.33.attention_norm.weight                  -> blk.33.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.33.ffn_norm.weight                        -> blk.33.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.34.attention.wq.weight                    -> blk.34.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.34.attention.wk.weight                    -> blk.34.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.34.attention.wv.weight                    -> blk.34.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.34.attention.wo.weight                    -> blk.34.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.34.feed_forward.w1.weight                 -> blk.34.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.34.feed_forward.w2.weight                 -> blk.34.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.34.feed_forward.w3.weight                 -> blk.34.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.34.attention_norm.weight                  -> blk.34.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.34.ffn_norm.weight                        -> blk.34.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.35.attention.wq.weight                    -> blk.35.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.35.attention.wk.weight                    -> blk.35.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.35.attention.wv.weight                    -> blk.35.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.35.attention.wo.weight                    -> blk.35.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.35.feed_forward.w1.weight                 -> blk.35.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.35.feed_forward.w2.weight                 -> blk.35.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.35.feed_forward.w3.weight                 -> blk.35.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.35.attention_norm.weight                  -> blk.35.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.35.ffn_norm.weight                        -> blk.35.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.36.attention.wq.weight                    -> blk.36.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.36.attention.wk.weight                    -> blk.36.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.36.attention.wv.weight                    -> blk.36.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.36.attention.wo.weight                    -> blk.36.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.36.feed_forward.w1.weight                 -> blk.36.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.36.feed_forward.w2.weight                 -> blk.36.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.36.feed_forward.w3.weight                 -> blk.36.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.36.attention_norm.weight                  -> blk.36.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.36.ffn_norm.weight                        -> blk.36.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.37.attention.wq.weight                    -> blk.37.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.37.attention.wk.weight                    -> blk.37.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.37.attention.wv.weight                    -> blk.37.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.37.attention.wo.weight                    -> blk.37.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.37.feed_forward.w1.weight                 -> blk.37.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.37.feed_forward.w2.weight                 -> blk.37.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.37.feed_forward.w3.weight                 -> blk.37.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.37.attention_norm.weight                  -> blk.37.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.37.ffn_norm.weight                        -> blk.37.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.38.attention.wq.weight                    -> blk.38.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.38.attention.wk.weight                    -> blk.38.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.38.attention.wv.weight                    -> blk.38.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.38.attention.wo.weight                    -> blk.38.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.38.feed_forward.w1.weight                 -> blk.38.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.38.feed_forward.w2.weight                 -> blk.38.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.38.feed_forward.w3.weight                 -> blk.38.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.38.attention_norm.weight                  -> blk.38.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.38.ffn_norm.weight                        -> blk.38.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.39.attention.wq.weight                    -> blk.39.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.39.attention.wk.weight                    -> blk.39.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.39.attention.wv.weight                    -> blk.39.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.39.attention.wo.weight                    -> blk.39.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.39.feed_forward.w1.weight                 -> blk.39.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.39.feed_forward.w2.weight                 -> blk.39.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.39.feed_forward.w3.weight                 -> blk.39.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.39.attention_norm.weight                  -> blk.39.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.39.ffn_norm.weight                        -> blk.39.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.40.attention.wq.weight                    -> blk.40.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.40.attention.wk.weight                    -> blk.40.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.40.attention.wv.weight                    -> blk.40.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.40.attention.wo.weight                    -> blk.40.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.40.feed_forward.w1.weight                 -> blk.40.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.40.feed_forward.w2.weight                 -> blk.40.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.40.feed_forward.w3.weight                 -> blk.40.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.40.attention_norm.weight                  -> blk.40.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.40.ffn_norm.weight                        -> blk.40.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.41.attention.wq.weight                    -> blk.41.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.41.attention.wk.weight                    -> blk.41.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.41.attention.wv.weight                    -> blk.41.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.41.attention.wo.weight                    -> blk.41.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.41.feed_forward.w1.weight                 -> blk.41.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.41.feed_forward.w2.weight                 -> blk.41.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.41.feed_forward.w3.weight                 -> blk.41.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.41.attention_norm.weight                  -> blk.41.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.41.ffn_norm.weight                        -> blk.41.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.42.attention.wq.weight                    -> blk.42.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.42.attention.wk.weight                    -> blk.42.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.42.attention.wv.weight                    -> blk.42.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.42.attention.wo.weight                    -> blk.42.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.42.feed_forward.w1.weight                 -> blk.42.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.42.feed_forward.w2.weight                 -> blk.42.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.42.feed_forward.w3.weight                 -> blk.42.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.42.attention_norm.weight                  -> blk.42.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.42.ffn_norm.weight                        -> blk.42.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.43.attention.wq.weight                    -> blk.43.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.43.attention.wk.weight                    -> blk.43.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.43.attention.wv.weight                    -> blk.43.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.43.attention.wo.weight                    -> blk.43.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.43.feed_forward.w1.weight                 -> blk.43.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.43.feed_forward.w2.weight                 -> blk.43.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.43.feed_forward.w3.weight                 -> blk.43.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.43.attention_norm.weight                  -> blk.43.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.43.ffn_norm.weight                        -> blk.43.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.44.attention.wq.weight                    -> blk.44.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.44.attention.wk.weight                    -> blk.44.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.44.attention.wv.weight                    -> blk.44.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.44.attention.wo.weight                    -> blk.44.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.44.feed_forward.w1.weight                 -> blk.44.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.44.feed_forward.w2.weight                 -> blk.44.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.44.feed_forward.w3.weight                 -> blk.44.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.44.attention_norm.weight                  -> blk.44.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.44.ffn_norm.weight                        -> blk.44.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.45.attention.wq.weight                    -> blk.45.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.45.attention.wk.weight                    -> blk.45.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.45.attention.wv.weight                    -> blk.45.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.45.attention.wo.weight                    -> blk.45.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.45.feed_forward.w1.weight                 -> blk.45.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.45.feed_forward.w2.weight                 -> blk.45.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.45.feed_forward.w3.weight                 -> blk.45.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.45.attention_norm.weight                  -> blk.45.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.45.ffn_norm.weight                        -> blk.45.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.46.attention.wq.weight                    -> blk.46.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.46.attention.wk.weight                    -> blk.46.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.46.attention.wv.weight                    -> blk.46.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.46.attention.wo.weight                    -> blk.46.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.46.feed_forward.w1.weight                 -> blk.46.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.46.feed_forward.w2.weight                 -> blk.46.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.46.feed_forward.w3.weight                 -> blk.46.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.46.attention_norm.weight                  -> blk.46.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.46.ffn_norm.weight                        -> blk.46.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.47.attention.wq.weight                    -> blk.47.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.47.attention.wk.weight                    -> blk.47.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.47.attention.wv.weight                    -> blk.47.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.47.attention.wo.weight                    -> blk.47.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.47.feed_forward.w1.weight                 -> blk.47.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.47.feed_forward.w2.weight                 -> blk.47.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.47.feed_forward.w3.weight                 -> blk.47.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.47.attention_norm.weight                  -> blk.47.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.47.ffn_norm.weight                        -> blk.47.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.48.attention.wq.weight                    -> blk.48.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.48.attention.wk.weight                    -> blk.48.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.48.attention.wv.weight                    -> blk.48.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.48.attention.wo.weight                    -> blk.48.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.48.feed_forward.w1.weight                 -> blk.48.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.48.feed_forward.w2.weight                 -> blk.48.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.48.feed_forward.w3.weight                 -> blk.48.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.48.attention_norm.weight                  -> blk.48.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.48.ffn_norm.weight                        -> blk.48.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.49.attention.wq.weight                    -> blk.49.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.49.attention.wk.weight                    -> blk.49.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.49.attention.wv.weight                    -> blk.49.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.49.attention.wo.weight                    -> blk.49.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.49.feed_forward.w1.weight                 -> blk.49.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.49.feed_forward.w2.weight                 -> blk.49.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.49.feed_forward.w3.weight                 -> blk.49.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.49.attention_norm.weight                  -> blk.49.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.49.ffn_norm.weight                        -> blk.49.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.50.attention.wq.weight                    -> blk.50.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.50.attention.wk.weight                    -> blk.50.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.50.attention.wv.weight                    -> blk.50.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.50.attention.wo.weight                    -> blk.50.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.50.feed_forward.w1.weight                 -> blk.50.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.50.feed_forward.w2.weight                 -> blk.50.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.50.feed_forward.w3.weight                 -> blk.50.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.50.attention_norm.weight                  -> blk.50.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.50.ffn_norm.weight                        -> blk.50.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.51.attention.wq.weight                    -> blk.51.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.51.attention.wk.weight                    -> blk.51.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.51.attention.wv.weight                    -> blk.51.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.51.attention.wo.weight                    -> blk.51.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.51.feed_forward.w1.weight                 -> blk.51.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.51.feed_forward.w2.weight                 -> blk.51.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.51.feed_forward.w3.weight                 -> blk.51.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.51.attention_norm.weight                  -> blk.51.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.51.ffn_norm.weight                        -> blk.51.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.52.attention.wq.weight                    -> blk.52.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.52.attention.wk.weight                    -> blk.52.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.52.attention.wv.weight                    -> blk.52.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.52.attention.wo.weight                    -> blk.52.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.52.feed_forward.w1.weight                 -> blk.52.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.52.feed_forward.w2.weight                 -> blk.52.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.52.feed_forward.w3.weight                 -> blk.52.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.52.attention_norm.weight                  -> blk.52.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.52.ffn_norm.weight                        -> blk.52.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.53.attention.wq.weight                    -> blk.53.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.53.attention.wk.weight                    -> blk.53.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.53.attention.wv.weight                    -> blk.53.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.53.attention.wo.weight                    -> blk.53.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.53.feed_forward.w1.weight                 -> blk.53.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.53.feed_forward.w2.weight                 -> blk.53.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.53.feed_forward.w3.weight                 -> blk.53.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.53.attention_norm.weight                  -> blk.53.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.53.ffn_norm.weight                        -> blk.53.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.54.attention.wq.weight                    -> blk.54.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.54.attention.wk.weight                    -> blk.54.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.54.attention.wv.weight                    -> blk.54.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.54.attention.wo.weight                    -> blk.54.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.54.feed_forward.w1.weight                 -> blk.54.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.54.feed_forward.w2.weight                 -> blk.54.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.54.feed_forward.w3.weight                 -> blk.54.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.54.attention_norm.weight                  -> blk.54.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.54.ffn_norm.weight                        -> blk.54.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.55.attention.wq.weight                    -> blk.55.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.55.attention.wk.weight                    -> blk.55.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.55.attention.wv.weight                    -> blk.55.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.55.attention.wo.weight                    -> blk.55.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.55.feed_forward.w1.weight                 -> blk.55.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.55.feed_forward.w2.weight                 -> blk.55.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.55.feed_forward.w3.weight                 -> blk.55.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.55.attention_norm.weight                  -> blk.55.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.55.ffn_norm.weight                        -> blk.55.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.56.attention.wq.weight                    -> blk.56.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.56.attention.wk.weight                    -> blk.56.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.56.attention.wv.weight                    -> blk.56.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.56.attention.wo.weight                    -> blk.56.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.56.feed_forward.w1.weight                 -> blk.56.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.56.feed_forward.w2.weight                 -> blk.56.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.56.feed_forward.w3.weight                 -> blk.56.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.56.attention_norm.weight                  -> blk.56.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.56.ffn_norm.weight                        -> blk.56.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.57.attention.wq.weight                    -> blk.57.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.57.attention.wk.weight                    -> blk.57.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.57.attention.wv.weight                    -> blk.57.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.57.attention.wo.weight                    -> blk.57.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.57.feed_forward.w1.weight                 -> blk.57.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.57.feed_forward.w2.weight                 -> blk.57.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.57.feed_forward.w3.weight                 -> blk.57.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.57.attention_norm.weight                  -> blk.57.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.57.ffn_norm.weight                        -> blk.57.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.58.attention.wq.weight                    -> blk.58.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.58.attention.wk.weight                    -> blk.58.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.58.attention.wv.weight                    -> blk.58.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.58.attention.wo.weight                    -> blk.58.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.58.feed_forward.w1.weight                 -> blk.58.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.58.feed_forward.w2.weight                 -> blk.58.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.58.feed_forward.w3.weight                 -> blk.58.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.58.attention_norm.weight                  -> blk.58.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.58.ffn_norm.weight                        -> blk.58.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.59.attention.wq.weight                    -> blk.59.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.59.attention.wk.weight                    -> blk.59.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.59.attention.wv.weight                    -> blk.59.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.59.attention.wo.weight                    -> blk.59.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.59.feed_forward.w1.weight                 -> blk.59.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.59.feed_forward.w2.weight                 -> blk.59.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.59.feed_forward.w3.weight                 -> blk.59.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.59.attention_norm.weight                  -> blk.59.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.59.ffn_norm.weight                        -> blk.59.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.60.attention.wq.weight                    -> blk.60.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.60.attention.wk.weight                    -> blk.60.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.60.attention.wv.weight                    -> blk.60.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.60.attention.wo.weight                    -> blk.60.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.60.feed_forward.w1.weight                 -> blk.60.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.60.feed_forward.w2.weight                 -> blk.60.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.60.feed_forward.w3.weight                 -> blk.60.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.60.attention_norm.weight                  -> blk.60.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.60.ffn_norm.weight                        -> blk.60.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.61.attention.wq.weight                    -> blk.61.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.61.attention.wk.weight                    -> blk.61.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.61.attention.wv.weight                    -> blk.61.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.61.attention.wo.weight                    -> blk.61.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.61.feed_forward.w1.weight                 -> blk.61.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.61.feed_forward.w2.weight                 -> blk.61.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.61.feed_forward.w3.weight                 -> blk.61.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.61.attention_norm.weight                  -> blk.61.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.61.ffn_norm.weight                        -> blk.61.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.62.attention.wq.weight                    -> blk.62.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.62.attention.wk.weight                    -> blk.62.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.62.attention.wv.weight                    -> blk.62.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.62.attention.wo.weight                    -> blk.62.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.62.feed_forward.w1.weight                 -> blk.62.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.62.feed_forward.w2.weight                 -> blk.62.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.62.feed_forward.w3.weight                 -> blk.62.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.62.attention_norm.weight                  -> blk.62.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.62.ffn_norm.weight                        -> blk.62.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.63.attention.wq.weight                    -> blk.63.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.63.attention.wk.weight                    -> blk.63.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.63.attention.wv.weight                    -> blk.63.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.63.attention.wo.weight                    -> blk.63.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.63.feed_forward.w1.weight                 -> blk.63.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.63.feed_forward.w2.weight                 -> blk.63.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.63.feed_forward.w3.weight                 -> blk.63.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.63.attention_norm.weight                  -> blk.63.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.63.ffn_norm.weight                        -> blk.63.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.64.attention.wq.weight                    -> blk.64.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.64.attention.wk.weight                    -> blk.64.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.64.attention.wv.weight                    -> blk.64.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.64.attention.wo.weight                    -> blk.64.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.64.feed_forward.w1.weight                 -> blk.64.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.64.feed_forward.w2.weight                 -> blk.64.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.64.feed_forward.w3.weight                 -> blk.64.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.64.attention_norm.weight                  -> blk.64.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.64.ffn_norm.weight                        -> blk.64.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.65.attention.wq.weight                    -> blk.65.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.65.attention.wk.weight                    -> blk.65.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.65.attention.wv.weight                    -> blk.65.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.65.attention.wo.weight                    -> blk.65.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.65.feed_forward.w1.weight                 -> blk.65.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.65.feed_forward.w2.weight                 -> blk.65.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.65.feed_forward.w3.weight                 -> blk.65.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.65.attention_norm.weight                  -> blk.65.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.65.ffn_norm.weight                        -> blk.65.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.66.attention.wq.weight                    -> blk.66.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.66.attention.wk.weight                    -> blk.66.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.66.attention.wv.weight                    -> blk.66.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.66.attention.wo.weight                    -> blk.66.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.66.feed_forward.w1.weight                 -> blk.66.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.66.feed_forward.w2.weight                 -> blk.66.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.66.feed_forward.w3.weight                 -> blk.66.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.66.attention_norm.weight                  -> blk.66.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.66.ffn_norm.weight                        -> blk.66.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.67.attention.wq.weight                    -> blk.67.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.67.attention.wk.weight                    -> blk.67.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.67.attention.wv.weight                    -> blk.67.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.67.attention.wo.weight                    -> blk.67.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.67.feed_forward.w1.weight                 -> blk.67.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.67.feed_forward.w2.weight                 -> blk.67.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.67.feed_forward.w3.weight                 -> blk.67.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.67.attention_norm.weight                  -> blk.67.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.67.ffn_norm.weight                        -> blk.67.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.68.attention.wq.weight                    -> blk.68.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.68.attention.wk.weight                    -> blk.68.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.68.attention.wv.weight                    -> blk.68.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.68.attention.wo.weight                    -> blk.68.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.68.feed_forward.w1.weight                 -> blk.68.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.68.feed_forward.w2.weight                 -> blk.68.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.68.feed_forward.w3.weight                 -> blk.68.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.68.attention_norm.weight                  -> blk.68.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.68.ffn_norm.weight                        -> blk.68.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.69.attention.wq.weight                    -> blk.69.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.69.attention.wk.weight                    -> blk.69.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.69.attention.wv.weight                    -> blk.69.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.69.attention.wo.weight                    -> blk.69.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.69.feed_forward.w1.weight                 -> blk.69.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.69.feed_forward.w2.weight                 -> blk.69.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.69.feed_forward.w3.weight                 -> blk.69.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.69.attention_norm.weight                  -> blk.69.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.69.ffn_norm.weight                        -> blk.69.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.70.attention.wq.weight                    -> blk.70.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.70.attention.wk.weight                    -> blk.70.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.70.attention.wv.weight                    -> blk.70.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.70.attention.wo.weight                    -> blk.70.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.70.feed_forward.w1.weight                 -> blk.70.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.70.feed_forward.w2.weight                 -> blk.70.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.70.feed_forward.w3.weight                 -> blk.70.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.70.attention_norm.weight                  -> blk.70.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.70.ffn_norm.weight                        -> blk.70.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.71.attention.wq.weight                    -> blk.71.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.71.attention.wk.weight                    -> blk.71.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.71.attention.wv.weight                    -> blk.71.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.71.attention.wo.weight                    -> blk.71.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.71.feed_forward.w1.weight                 -> blk.71.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.71.feed_forward.w2.weight                 -> blk.71.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.71.feed_forward.w3.weight                 -> blk.71.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.71.attention_norm.weight                  -> blk.71.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.71.ffn_norm.weight                        -> blk.71.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.72.attention.wq.weight                    -> blk.72.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.72.attention.wk.weight                    -> blk.72.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.72.attention.wv.weight                    -> blk.72.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.72.attention.wo.weight                    -> blk.72.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.72.feed_forward.w1.weight                 -> blk.72.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.72.feed_forward.w2.weight                 -> blk.72.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.72.feed_forward.w3.weight                 -> blk.72.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.72.attention_norm.weight                  -> blk.72.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.72.ffn_norm.weight                        -> blk.72.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.73.attention.wq.weight                    -> blk.73.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.73.attention.wk.weight                    -> blk.73.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.73.attention.wv.weight                    -> blk.73.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.73.attention.wo.weight                    -> blk.73.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.73.feed_forward.w1.weight                 -> blk.73.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.73.feed_forward.w2.weight                 -> blk.73.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.73.feed_forward.w3.weight                 -> blk.73.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.73.attention_norm.weight                  -> blk.73.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.73.ffn_norm.weight                        -> blk.73.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.74.attention.wq.weight                    -> blk.74.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.74.attention.wk.weight                    -> blk.74.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.74.attention.wv.weight                    -> blk.74.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.74.attention.wo.weight                    -> blk.74.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.74.feed_forward.w1.weight                 -> blk.74.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.74.feed_forward.w2.weight                 -> blk.74.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.74.feed_forward.w3.weight                 -> blk.74.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.74.attention_norm.weight                  -> blk.74.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.74.ffn_norm.weight                        -> blk.74.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.75.attention.wq.weight                    -> blk.75.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.75.attention.wk.weight                    -> blk.75.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.75.attention.wv.weight                    -> blk.75.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.75.attention.wo.weight                    -> blk.75.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.75.feed_forward.w1.weight                 -> blk.75.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.75.feed_forward.w2.weight                 -> blk.75.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.75.feed_forward.w3.weight                 -> blk.75.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.75.attention_norm.weight                  -> blk.75.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.75.ffn_norm.weight                        -> blk.75.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.76.attention.wq.weight                    -> blk.76.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.76.attention.wk.weight                    -> blk.76.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.76.attention.wv.weight                    -> blk.76.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.76.attention.wo.weight                    -> blk.76.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.76.feed_forward.w1.weight                 -> blk.76.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.76.feed_forward.w2.weight                 -> blk.76.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.76.feed_forward.w3.weight                 -> blk.76.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.76.attention_norm.weight                  -> blk.76.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.76.ffn_norm.weight                        -> blk.76.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.77.attention.wq.weight                    -> blk.77.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.77.attention.wk.weight                    -> blk.77.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.77.attention.wv.weight                    -> blk.77.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.77.attention.wo.weight                    -> blk.77.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.77.feed_forward.w1.weight                 -> blk.77.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.77.feed_forward.w2.weight                 -> blk.77.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.77.feed_forward.w3.weight                 -> blk.77.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.77.attention_norm.weight                  -> blk.77.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.77.ffn_norm.weight                        -> blk.77.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.78.attention.wq.weight                    -> blk.78.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.78.attention.wk.weight                    -> blk.78.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.78.attention.wv.weight                    -> blk.78.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.78.attention.wo.weight                    -> blk.78.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.78.feed_forward.w1.weight                 -> blk.78.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.78.feed_forward.w2.weight                 -> blk.78.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.78.feed_forward.w3.weight                 -> blk.78.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.78.attention_norm.weight                  -> blk.78.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.78.ffn_norm.weight                        -> blk.78.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.79.attention.wq.weight                    -> blk.79.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.79.attention.wk.weight                    -> blk.79.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.79.attention.wv.weight                    -> blk.79.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.79.attention.wo.weight                    -> blk.79.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.79.feed_forward.w1.weight                 -> blk.79.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.79.feed_forward.w2.weight                 -> blk.79.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.79.feed_forward.w3.weight                 -> blk.79.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.79.attention_norm.weight                  -> blk.79.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.79.ffn_norm.weight                        -> blk.79.ffn_norm.weight                   | BF16   | [8192]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/70B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/723] Writing tensor token_embd.weight                      | size  32000 x   8192  | type F16  | T+   2\n",
      "[  2/723] Writing tensor output_norm.weight                     | size   8192           | type F32  | T+   2\n",
      "[  3/723] Writing tensor output.weight                          | size  32000 x   8192  | type F16  | T+   2\n",
      "[  4/723] Writing tensor blk.0.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   2\n",
      "[  5/723] Writing tensor blk.0.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   2\n",
      "[  6/723] Writing tensor blk.0.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   2\n",
      "[  7/723] Writing tensor blk.0.attn_output.weight               | size   8192 x   8192  | type F16  | T+   2\n",
      "[  8/723] Writing tensor blk.0.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   2\n",
      "[  9/723] Writing tensor blk.0.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   3\n",
      "[ 10/723] Writing tensor blk.0.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   4\n",
      "[ 11/723] Writing tensor blk.0.attn_norm.weight                 | size   8192           | type F32  | T+   4\n",
      "[ 12/723] Writing tensor blk.0.ffn_norm.weight                  | size   8192           | type F32  | T+   4\n",
      "[ 13/723] Writing tensor blk.1.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   4\n",
      "[ 14/723] Writing tensor blk.1.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   4\n",
      "[ 15/723] Writing tensor blk.1.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   4\n",
      "[ 16/723] Writing tensor blk.1.attn_output.weight               | size   8192 x   8192  | type F16  | T+   4\n",
      "[ 17/723] Writing tensor blk.1.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   5\n",
      "[ 18/723] Writing tensor blk.1.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   5\n",
      "[ 19/723] Writing tensor blk.1.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   6\n",
      "[ 20/723] Writing tensor blk.1.attn_norm.weight                 | size   8192           | type F32  | T+   6\n",
      "[ 21/723] Writing tensor blk.1.ffn_norm.weight                  | size   8192           | type F32  | T+   6\n",
      "[ 22/723] Writing tensor blk.2.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   6\n",
      "[ 23/723] Writing tensor blk.2.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   6\n",
      "[ 24/723] Writing tensor blk.2.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   6\n",
      "[ 25/723] Writing tensor blk.2.attn_output.weight               | size   8192 x   8192  | type F16  | T+   6\n",
      "[ 26/723] Writing tensor blk.2.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   7\n",
      "[ 27/723] Writing tensor blk.2.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   7\n",
      "[ 28/723] Writing tensor blk.2.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   8\n",
      "[ 29/723] Writing tensor blk.2.attn_norm.weight                 | size   8192           | type F32  | T+   8\n",
      "[ 30/723] Writing tensor blk.2.ffn_norm.weight                  | size   8192           | type F32  | T+   8\n",
      "[ 31/723] Writing tensor blk.3.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   8\n",
      "[ 32/723] Writing tensor blk.3.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   8\n",
      "[ 33/723] Writing tensor blk.3.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   8\n",
      "[ 34/723] Writing tensor blk.3.attn_output.weight               | size   8192 x   8192  | type F16  | T+   8\n",
      "[ 35/723] Writing tensor blk.3.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   9\n",
      "[ 36/723] Writing tensor blk.3.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   9\n",
      "[ 37/723] Writing tensor blk.3.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  10\n",
      "[ 38/723] Writing tensor blk.3.attn_norm.weight                 | size   8192           | type F32  | T+  10\n",
      "[ 39/723] Writing tensor blk.3.ffn_norm.weight                  | size   8192           | type F32  | T+  10\n",
      "[ 40/723] Writing tensor blk.4.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  10\n",
      "[ 41/723] Writing tensor blk.4.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  10\n",
      "[ 42/723] Writing tensor blk.4.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  10\n",
      "[ 43/723] Writing tensor blk.4.attn_output.weight               | size   8192 x   8192  | type F16  | T+  10\n",
      "[ 44/723] Writing tensor blk.4.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  11\n",
      "[ 45/723] Writing tensor blk.4.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  12\n",
      "[ 46/723] Writing tensor blk.4.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  12\n",
      "[ 47/723] Writing tensor blk.4.attn_norm.weight                 | size   8192           | type F32  | T+  12\n",
      "[ 48/723] Writing tensor blk.4.ffn_norm.weight                  | size   8192           | type F32  | T+  12\n",
      "[ 49/723] Writing tensor blk.5.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  12\n",
      "[ 50/723] Writing tensor blk.5.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  12\n",
      "[ 51/723] Writing tensor blk.5.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  12\n",
      "[ 52/723] Writing tensor blk.5.attn_output.weight               | size   8192 x   8192  | type F16  | T+  12\n",
      "[ 53/723] Writing tensor blk.5.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  13\n",
      "[ 54/723] Writing tensor blk.5.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  14\n",
      "[ 55/723] Writing tensor blk.5.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  14\n",
      "[ 56/723] Writing tensor blk.5.attn_norm.weight                 | size   8192           | type F32  | T+  14\n",
      "[ 57/723] Writing tensor blk.5.ffn_norm.weight                  | size   8192           | type F32  | T+  14\n",
      "[ 58/723] Writing tensor blk.6.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  14\n",
      "[ 59/723] Writing tensor blk.6.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  14\n",
      "[ 60/723] Writing tensor blk.6.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  14\n",
      "[ 61/723] Writing tensor blk.6.attn_output.weight               | size   8192 x   8192  | type F16  | T+  14\n",
      "[ 62/723] Writing tensor blk.6.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  15\n",
      "[ 63/723] Writing tensor blk.6.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  16\n",
      "[ 64/723] Writing tensor blk.6.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  16\n",
      "[ 65/723] Writing tensor blk.6.attn_norm.weight                 | size   8192           | type F32  | T+  16\n",
      "[ 66/723] Writing tensor blk.6.ffn_norm.weight                  | size   8192           | type F32  | T+  16\n",
      "[ 67/723] Writing tensor blk.7.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  16\n",
      "[ 68/723] Writing tensor blk.7.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  16\n",
      "[ 69/723] Writing tensor blk.7.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  16\n",
      "[ 70/723] Writing tensor blk.7.attn_output.weight               | size   8192 x   8192  | type F16  | T+  16\n",
      "[ 71/723] Writing tensor blk.7.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  18\n",
      "[ 72/723] Writing tensor blk.7.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  18\n",
      "[ 73/723] Writing tensor blk.7.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  18\n",
      "[ 74/723] Writing tensor blk.7.attn_norm.weight                 | size   8192           | type F32  | T+  18\n",
      "[ 75/723] Writing tensor blk.7.ffn_norm.weight                  | size   8192           | type F32  | T+  18\n",
      "[ 76/723] Writing tensor blk.8.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  18\n",
      "[ 77/723] Writing tensor blk.8.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  18\n",
      "[ 78/723] Writing tensor blk.8.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  18\n",
      "[ 79/723] Writing tensor blk.8.attn_output.weight               | size   8192 x   8192  | type F16  | T+  18\n",
      "[ 80/723] Writing tensor blk.8.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  20\n",
      "[ 81/723] Writing tensor blk.8.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  20\n",
      "[ 82/723] Writing tensor blk.8.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  20\n",
      "[ 83/723] Writing tensor blk.8.attn_norm.weight                 | size   8192           | type F32  | T+  20\n",
      "[ 84/723] Writing tensor blk.8.ffn_norm.weight                  | size   8192           | type F32  | T+  20\n",
      "[ 85/723] Writing tensor blk.9.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  20\n",
      "[ 86/723] Writing tensor blk.9.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  20\n",
      "[ 87/723] Writing tensor blk.9.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  20\n",
      "[ 88/723] Writing tensor blk.9.attn_output.weight               | size   8192 x   8192  | type F16  | T+  20\n",
      "[ 89/723] Writing tensor blk.9.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  22\n",
      "[ 90/723] Writing tensor blk.9.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  22\n",
      "[ 91/723] Writing tensor blk.9.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  22\n",
      "[ 92/723] Writing tensor blk.9.attn_norm.weight                 | size   8192           | type F32  | T+  22\n",
      "[ 93/723] Writing tensor blk.9.ffn_norm.weight                  | size   8192           | type F32  | T+  22\n",
      "[ 94/723] Writing tensor blk.10.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  22\n",
      "[ 95/723] Writing tensor blk.10.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  22\n",
      "[ 96/723] Writing tensor blk.10.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  22\n",
      "[ 97/723] Writing tensor blk.10.attn_output.weight              | size   8192 x   8192  | type F16  | T+  22\n",
      "[ 98/723] Writing tensor blk.10.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  24\n",
      "[ 99/723] Writing tensor blk.10.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  24\n",
      "[100/723] Writing tensor blk.10.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  24\n",
      "[101/723] Writing tensor blk.10.attn_norm.weight                | size   8192           | type F32  | T+  24\n",
      "[102/723] Writing tensor blk.10.ffn_norm.weight                 | size   8192           | type F32  | T+  24\n",
      "[103/723] Writing tensor blk.11.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  24\n",
      "[104/723] Writing tensor blk.11.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  24\n",
      "[105/723] Writing tensor blk.11.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  24\n",
      "[106/723] Writing tensor blk.11.attn_output.weight              | size   8192 x   8192  | type F16  | T+  24\n",
      "[107/723] Writing tensor blk.11.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  26\n",
      "[108/723] Writing tensor blk.11.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  26\n",
      "[109/723] Writing tensor blk.11.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  26\n",
      "[110/723] Writing tensor blk.11.attn_norm.weight                | size   8192           | type F32  | T+  26\n",
      "[111/723] Writing tensor blk.11.ffn_norm.weight                 | size   8192           | type F32  | T+  26\n",
      "[112/723] Writing tensor blk.12.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  26\n",
      "[113/723] Writing tensor blk.12.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  26\n",
      "[114/723] Writing tensor blk.12.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  26\n",
      "[115/723] Writing tensor blk.12.attn_output.weight              | size   8192 x   8192  | type F16  | T+  26\n",
      "[116/723] Writing tensor blk.12.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  28\n",
      "[117/723] Writing tensor blk.12.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  28\n",
      "[118/723] Writing tensor blk.12.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  28\n",
      "[119/723] Writing tensor blk.12.attn_norm.weight                | size   8192           | type F32  | T+  28\n",
      "[120/723] Writing tensor blk.12.ffn_norm.weight                 | size   8192           | type F32  | T+  28\n",
      "[121/723] Writing tensor blk.13.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  28\n",
      "[122/723] Writing tensor blk.13.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  28\n",
      "[123/723] Writing tensor blk.13.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  28\n",
      "[124/723] Writing tensor blk.13.attn_output.weight              | size   8192 x   8192  | type F16  | T+  28\n",
      "[125/723] Writing tensor blk.13.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  30\n",
      "[126/723] Writing tensor blk.13.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  30\n",
      "[127/723] Writing tensor blk.13.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  31\n",
      "[128/723] Writing tensor blk.13.attn_norm.weight                | size   8192           | type F32  | T+  31\n",
      "[129/723] Writing tensor blk.13.ffn_norm.weight                 | size   8192           | type F32  | T+  31\n",
      "[130/723] Writing tensor blk.14.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  31\n",
      "[131/723] Writing tensor blk.14.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  31\n",
      "[132/723] Writing tensor blk.14.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  31\n",
      "[133/723] Writing tensor blk.14.attn_output.weight              | size   8192 x   8192  | type F16  | T+  31\n",
      "[134/723] Writing tensor blk.14.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  32\n",
      "[135/723] Writing tensor blk.14.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  33\n",
      "[136/723] Writing tensor blk.14.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  33\n",
      "[137/723] Writing tensor blk.14.attn_norm.weight                | size   8192           | type F32  | T+  33\n",
      "[138/723] Writing tensor blk.14.ffn_norm.weight                 | size   8192           | type F32  | T+  33\n",
      "[139/723] Writing tensor blk.15.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  33\n",
      "[140/723] Writing tensor blk.15.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  33\n",
      "[141/723] Writing tensor blk.15.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  33\n",
      "[142/723] Writing tensor blk.15.attn_output.weight              | size   8192 x   8192  | type F16  | T+  33\n",
      "[143/723] Writing tensor blk.15.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  35\n",
      "[144/723] Writing tensor blk.15.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  35\n",
      "[145/723] Writing tensor blk.15.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  35\n",
      "[146/723] Writing tensor blk.15.attn_norm.weight                | size   8192           | type F32  | T+  36\n",
      "[147/723] Writing tensor blk.15.ffn_norm.weight                 | size   8192           | type F32  | T+  36\n",
      "[148/723] Writing tensor blk.16.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  36\n",
      "[149/723] Writing tensor blk.16.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  36\n",
      "[150/723] Writing tensor blk.16.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  36\n",
      "[151/723] Writing tensor blk.16.attn_output.weight              | size   8192 x   8192  | type F16  | T+  36\n",
      "[152/723] Writing tensor blk.16.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  38\n",
      "[153/723] Writing tensor blk.16.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  38\n",
      "[154/723] Writing tensor blk.16.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  38\n",
      "[155/723] Writing tensor blk.16.attn_norm.weight                | size   8192           | type F32  | T+  39\n",
      "[156/723] Writing tensor blk.16.ffn_norm.weight                 | size   8192           | type F32  | T+  39\n",
      "[157/723] Writing tensor blk.17.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  39\n",
      "[158/723] Writing tensor blk.17.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  39\n",
      "[159/723] Writing tensor blk.17.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  39\n",
      "[160/723] Writing tensor blk.17.attn_output.weight              | size   8192 x   8192  | type F16  | T+  39\n",
      "[161/723] Writing tensor blk.17.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  40\n",
      "[162/723] Writing tensor blk.17.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  40\n",
      "[163/723] Writing tensor blk.17.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  41\n",
      "[164/723] Writing tensor blk.17.attn_norm.weight                | size   8192           | type F32  | T+  41\n",
      "[165/723] Writing tensor blk.17.ffn_norm.weight                 | size   8192           | type F32  | T+  41\n",
      "[166/723] Writing tensor blk.18.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  41\n",
      "[167/723] Writing tensor blk.18.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  41\n",
      "[168/723] Writing tensor blk.18.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  41\n",
      "[169/723] Writing tensor blk.18.attn_output.weight              | size   8192 x   8192  | type F16  | T+  41\n",
      "[170/723] Writing tensor blk.18.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  42\n",
      "[171/723] Writing tensor blk.18.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  43\n",
      "[172/723] Writing tensor blk.18.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  43\n",
      "[173/723] Writing tensor blk.18.attn_norm.weight                | size   8192           | type F32  | T+  43\n",
      "[174/723] Writing tensor blk.18.ffn_norm.weight                 | size   8192           | type F32  | T+  43\n",
      "[175/723] Writing tensor blk.19.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  43\n",
      "[176/723] Writing tensor blk.19.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  43\n",
      "[177/723] Writing tensor blk.19.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  43\n",
      "[178/723] Writing tensor blk.19.attn_output.weight              | size   8192 x   8192  | type F16  | T+  43\n",
      "[179/723] Writing tensor blk.19.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  45\n",
      "[180/723] Writing tensor blk.19.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  45\n",
      "[181/723] Writing tensor blk.19.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  45\n",
      "[182/723] Writing tensor blk.19.attn_norm.weight                | size   8192           | type F32  | T+  45\n",
      "[183/723] Writing tensor blk.19.ffn_norm.weight                 | size   8192           | type F32  | T+  45\n",
      "[184/723] Writing tensor blk.20.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  45\n",
      "[185/723] Writing tensor blk.20.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  45\n",
      "[186/723] Writing tensor blk.20.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  45\n",
      "[187/723] Writing tensor blk.20.attn_output.weight              | size   8192 x   8192  | type F16  | T+  45\n",
      "[188/723] Writing tensor blk.20.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  47\n",
      "[189/723] Writing tensor blk.20.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  47\n",
      "[190/723] Writing tensor blk.20.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  47\n",
      "[191/723] Writing tensor blk.20.attn_norm.weight                | size   8192           | type F32  | T+  47\n",
      "[192/723] Writing tensor blk.20.ffn_norm.weight                 | size   8192           | type F32  | T+  47\n",
      "[193/723] Writing tensor blk.21.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  47\n",
      "[194/723] Writing tensor blk.21.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  47\n",
      "[195/723] Writing tensor blk.21.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  47\n",
      "[196/723] Writing tensor blk.21.attn_output.weight              | size   8192 x   8192  | type F16  | T+  47\n",
      "[197/723] Writing tensor blk.21.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  49\n",
      "[198/723] Writing tensor blk.21.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  49\n",
      "[199/723] Writing tensor blk.21.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  50\n",
      "[200/723] Writing tensor blk.21.attn_norm.weight                | size   8192           | type F32  | T+  50\n",
      "[201/723] Writing tensor blk.21.ffn_norm.weight                 | size   8192           | type F32  | T+  50\n",
      "[202/723] Writing tensor blk.22.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  50\n",
      "[203/723] Writing tensor blk.22.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  50\n",
      "[204/723] Writing tensor blk.22.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  50\n",
      "[205/723] Writing tensor blk.22.attn_output.weight              | size   8192 x   8192  | type F16  | T+  50\n",
      "[206/723] Writing tensor blk.22.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  52\n",
      "[207/723] Writing tensor blk.22.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  52\n",
      "[208/723] Writing tensor blk.22.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  52\n",
      "[209/723] Writing tensor blk.22.attn_norm.weight                | size   8192           | type F32  | T+  52\n",
      "[210/723] Writing tensor blk.22.ffn_norm.weight                 | size   8192           | type F32  | T+  52\n",
      "[211/723] Writing tensor blk.23.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  52\n",
      "[212/723] Writing tensor blk.23.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  52\n",
      "[213/723] Writing tensor blk.23.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  52\n",
      "[214/723] Writing tensor blk.23.attn_output.weight              | size   8192 x   8192  | type F16  | T+  52\n",
      "[215/723] Writing tensor blk.23.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  54\n",
      "[216/723] Writing tensor blk.23.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  54\n",
      "[217/723] Writing tensor blk.23.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  55\n",
      "[218/723] Writing tensor blk.23.attn_norm.weight                | size   8192           | type F32  | T+  55\n",
      "[219/723] Writing tensor blk.23.ffn_norm.weight                 | size   8192           | type F32  | T+  55\n",
      "[220/723] Writing tensor blk.24.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  55\n",
      "[221/723] Writing tensor blk.24.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  55\n",
      "[222/723] Writing tensor blk.24.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  55\n",
      "[223/723] Writing tensor blk.24.attn_output.weight              | size   8192 x   8192  | type F16  | T+  55\n",
      "[224/723] Writing tensor blk.24.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  56\n",
      "[225/723] Writing tensor blk.24.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  57\n",
      "[226/723] Writing tensor blk.24.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  57\n",
      "[227/723] Writing tensor blk.24.attn_norm.weight                | size   8192           | type F32  | T+  57\n",
      "[228/723] Writing tensor blk.24.ffn_norm.weight                 | size   8192           | type F32  | T+  57\n",
      "[229/723] Writing tensor blk.25.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  57\n",
      "[230/723] Writing tensor blk.25.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  57\n",
      "[231/723] Writing tensor blk.25.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  57\n",
      "[232/723] Writing tensor blk.25.attn_output.weight              | size   8192 x   8192  | type F16  | T+  57\n",
      "[233/723] Writing tensor blk.25.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  59\n",
      "[234/723] Writing tensor blk.25.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  59\n",
      "[235/723] Writing tensor blk.25.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  59\n",
      "[236/723] Writing tensor blk.25.attn_norm.weight                | size   8192           | type F32  | T+  59\n",
      "[237/723] Writing tensor blk.25.ffn_norm.weight                 | size   8192           | type F32  | T+  59\n",
      "[238/723] Writing tensor blk.26.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  59\n",
      "[239/723] Writing tensor blk.26.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  59\n",
      "[240/723] Writing tensor blk.26.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  59\n",
      "[241/723] Writing tensor blk.26.attn_output.weight              | size   8192 x   8192  | type F16  | T+  59\n",
      "[242/723] Writing tensor blk.26.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  61\n",
      "[243/723] Writing tensor blk.26.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  61\n",
      "[244/723] Writing tensor blk.26.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  61\n",
      "[245/723] Writing tensor blk.26.attn_norm.weight                | size   8192           | type F32  | T+  61\n",
      "[246/723] Writing tensor blk.26.ffn_norm.weight                 | size   8192           | type F32  | T+  61\n",
      "[247/723] Writing tensor blk.27.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  61\n",
      "[248/723] Writing tensor blk.27.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  61\n",
      "[249/723] Writing tensor blk.27.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  61\n",
      "[250/723] Writing tensor blk.27.attn_output.weight              | size   8192 x   8192  | type F16  | T+  61\n",
      "[251/723] Writing tensor blk.27.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  63\n",
      "[252/723] Writing tensor blk.27.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  63\n",
      "[253/723] Writing tensor blk.27.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  63\n",
      "[254/723] Writing tensor blk.27.attn_norm.weight                | size   8192           | type F32  | T+  64\n",
      "[255/723] Writing tensor blk.27.ffn_norm.weight                 | size   8192           | type F32  | T+  64\n",
      "[256/723] Writing tensor blk.28.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  64\n",
      "[257/723] Writing tensor blk.28.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  64\n",
      "[258/723] Writing tensor blk.28.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  64\n",
      "[259/723] Writing tensor blk.28.attn_output.weight              | size   8192 x   8192  | type F16  | T+  64\n",
      "[260/723] Writing tensor blk.28.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  65\n",
      "[261/723] Writing tensor blk.28.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  66\n",
      "[262/723] Writing tensor blk.28.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  66\n",
      "[263/723] Writing tensor blk.28.attn_norm.weight                | size   8192           | type F32  | T+  66\n",
      "[264/723] Writing tensor blk.28.ffn_norm.weight                 | size   8192           | type F32  | T+  66\n",
      "[265/723] Writing tensor blk.29.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  66\n",
      "[266/723] Writing tensor blk.29.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  66\n",
      "[267/723] Writing tensor blk.29.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  66\n",
      "[268/723] Writing tensor blk.29.attn_output.weight              | size   8192 x   8192  | type F16  | T+  66\n",
      "[269/723] Writing tensor blk.29.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  68\n",
      "[270/723] Writing tensor blk.29.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  68\n",
      "[271/723] Writing tensor blk.29.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  68\n",
      "[272/723] Writing tensor blk.29.attn_norm.weight                | size   8192           | type F32  | T+  68\n",
      "[273/723] Writing tensor blk.29.ffn_norm.weight                 | size   8192           | type F32  | T+  68\n",
      "[274/723] Writing tensor blk.30.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  68\n",
      "[275/723] Writing tensor blk.30.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  68\n",
      "[276/723] Writing tensor blk.30.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  68\n",
      "[277/723] Writing tensor blk.30.attn_output.weight              | size   8192 x   8192  | type F16  | T+  68\n",
      "[278/723] Writing tensor blk.30.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  70\n",
      "[279/723] Writing tensor blk.30.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  70\n",
      "[280/723] Writing tensor blk.30.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  70\n",
      "[281/723] Writing tensor blk.30.attn_norm.weight                | size   8192           | type F32  | T+  70\n",
      "[282/723] Writing tensor blk.30.ffn_norm.weight                 | size   8192           | type F32  | T+  70\n",
      "[283/723] Writing tensor blk.31.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  70\n",
      "[284/723] Writing tensor blk.31.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  70\n",
      "[285/723] Writing tensor blk.31.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  70\n",
      "[286/723] Writing tensor blk.31.attn_output.weight              | size   8192 x   8192  | type F16  | T+  70\n",
      "[287/723] Writing tensor blk.31.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  72\n",
      "[288/723] Writing tensor blk.31.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  72\n",
      "[289/723] Writing tensor blk.31.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  72\n",
      "[290/723] Writing tensor blk.31.attn_norm.weight                | size   8192           | type F32  | T+  73\n",
      "[291/723] Writing tensor blk.31.ffn_norm.weight                 | size   8192           | type F32  | T+  73\n",
      "[292/723] Writing tensor blk.32.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  73\n",
      "[293/723] Writing tensor blk.32.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  73\n",
      "[294/723] Writing tensor blk.32.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  73\n",
      "[295/723] Writing tensor blk.32.attn_output.weight              | size   8192 x   8192  | type F16  | T+  73\n",
      "[296/723] Writing tensor blk.32.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  74\n",
      "[297/723] Writing tensor blk.32.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  74\n",
      "[298/723] Writing tensor blk.32.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  75\n",
      "[299/723] Writing tensor blk.32.attn_norm.weight                | size   8192           | type F32  | T+  75\n",
      "[300/723] Writing tensor blk.32.ffn_norm.weight                 | size   8192           | type F32  | T+  75\n",
      "[301/723] Writing tensor blk.33.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  75\n",
      "[302/723] Writing tensor blk.33.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  75\n",
      "[303/723] Writing tensor blk.33.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  75\n",
      "[304/723] Writing tensor blk.33.attn_output.weight              | size   8192 x   8192  | type F16  | T+  75\n",
      "[305/723] Writing tensor blk.33.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  77\n",
      "[306/723] Writing tensor blk.33.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  77\n",
      "[307/723] Writing tensor blk.33.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  77\n",
      "[308/723] Writing tensor blk.33.attn_norm.weight                | size   8192           | type F32  | T+  77\n",
      "[309/723] Writing tensor blk.33.ffn_norm.weight                 | size   8192           | type F32  | T+  77\n",
      "[310/723] Writing tensor blk.34.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  77\n",
      "[311/723] Writing tensor blk.34.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  77\n",
      "[312/723] Writing tensor blk.34.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  77\n",
      "[313/723] Writing tensor blk.34.attn_output.weight              | size   8192 x   8192  | type F16  | T+  77\n",
      "[314/723] Writing tensor blk.34.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  79\n",
      "[315/723] Writing tensor blk.34.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  79\n",
      "[316/723] Writing tensor blk.34.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  79\n",
      "[317/723] Writing tensor blk.34.attn_norm.weight                | size   8192           | type F32  | T+  79\n",
      "[318/723] Writing tensor blk.34.ffn_norm.weight                 | size   8192           | type F32  | T+  79\n",
      "[319/723] Writing tensor blk.35.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  79\n",
      "[320/723] Writing tensor blk.35.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  80\n",
      "[321/723] Writing tensor blk.35.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  80\n",
      "[322/723] Writing tensor blk.35.attn_output.weight              | size   8192 x   8192  | type F16  | T+  80\n",
      "[323/723] Writing tensor blk.35.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  81\n",
      "[324/723] Writing tensor blk.35.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  81\n",
      "[325/723] Writing tensor blk.35.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  82\n",
      "[326/723] Writing tensor blk.35.attn_norm.weight                | size   8192           | type F32  | T+  82\n",
      "[327/723] Writing tensor blk.35.ffn_norm.weight                 | size   8192           | type F32  | T+  82\n",
      "[328/723] Writing tensor blk.36.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  82\n",
      "[329/723] Writing tensor blk.36.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  82\n",
      "[330/723] Writing tensor blk.36.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  82\n",
      "[331/723] Writing tensor blk.36.attn_output.weight              | size   8192 x   8192  | type F16  | T+  82\n",
      "[332/723] Writing tensor blk.36.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  83\n",
      "[333/723] Writing tensor blk.36.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  84\n",
      "[334/723] Writing tensor blk.36.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  84\n",
      "[335/723] Writing tensor blk.36.attn_norm.weight                | size   8192           | type F32  | T+  84\n",
      "[336/723] Writing tensor blk.36.ffn_norm.weight                 | size   8192           | type F32  | T+  84\n",
      "[337/723] Writing tensor blk.37.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  84\n",
      "[338/723] Writing tensor blk.37.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  85\n",
      "[339/723] Writing tensor blk.37.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  85\n",
      "[340/723] Writing tensor blk.37.attn_output.weight              | size   8192 x   8192  | type F16  | T+  85\n",
      "[341/723] Writing tensor blk.37.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  86\n",
      "[342/723] Writing tensor blk.37.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  86\n",
      "[343/723] Writing tensor blk.37.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  86\n",
      "[344/723] Writing tensor blk.37.attn_norm.weight                | size   8192           | type F32  | T+  87\n",
      "[345/723] Writing tensor blk.37.ffn_norm.weight                 | size   8192           | type F32  | T+  87\n",
      "[346/723] Writing tensor blk.38.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  87\n",
      "[347/723] Writing tensor blk.38.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  87\n",
      "[348/723] Writing tensor blk.38.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  87\n",
      "[349/723] Writing tensor blk.38.attn_output.weight              | size   8192 x   8192  | type F16  | T+  87\n",
      "[350/723] Writing tensor blk.38.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  88\n",
      "[351/723] Writing tensor blk.38.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  88\n",
      "[352/723] Writing tensor blk.38.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  89\n",
      "[353/723] Writing tensor blk.38.attn_norm.weight                | size   8192           | type F32  | T+  89\n",
      "[354/723] Writing tensor blk.38.ffn_norm.weight                 | size   8192           | type F32  | T+  89\n",
      "[355/723] Writing tensor blk.39.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  89\n",
      "[356/723] Writing tensor blk.39.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  89\n",
      "[357/723] Writing tensor blk.39.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  89\n",
      "[358/723] Writing tensor blk.39.attn_output.weight              | size   8192 x   8192  | type F16  | T+  89\n",
      "[359/723] Writing tensor blk.39.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  90\n",
      "[360/723] Writing tensor blk.39.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  91\n",
      "[361/723] Writing tensor blk.39.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  91\n",
      "[362/723] Writing tensor blk.39.attn_norm.weight                | size   8192           | type F32  | T+  91\n",
      "[363/723] Writing tensor blk.39.ffn_norm.weight                 | size   8192           | type F32  | T+  91\n",
      "[364/723] Writing tensor blk.40.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  91\n",
      "[365/723] Writing tensor blk.40.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  91\n",
      "[366/723] Writing tensor blk.40.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  91\n",
      "[367/723] Writing tensor blk.40.attn_output.weight              | size   8192 x   8192  | type F16  | T+  91\n",
      "[368/723] Writing tensor blk.40.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  93\n",
      "[369/723] Writing tensor blk.40.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  93\n",
      "[370/723] Writing tensor blk.40.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  93\n",
      "[371/723] Writing tensor blk.40.attn_norm.weight                | size   8192           | type F32  | T+  93\n",
      "[372/723] Writing tensor blk.40.ffn_norm.weight                 | size   8192           | type F32  | T+  93\n",
      "[373/723] Writing tensor blk.41.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  93\n",
      "[374/723] Writing tensor blk.41.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  93\n",
      "[375/723] Writing tensor blk.41.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  93\n",
      "[376/723] Writing tensor blk.41.attn_output.weight              | size   8192 x   8192  | type F16  | T+  93\n",
      "[377/723] Writing tensor blk.41.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  95\n",
      "[378/723] Writing tensor blk.41.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  95\n",
      "[379/723] Writing tensor blk.41.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  95\n",
      "[380/723] Writing tensor blk.41.attn_norm.weight                | size   8192           | type F32  | T+  95\n",
      "[381/723] Writing tensor blk.41.ffn_norm.weight                 | size   8192           | type F32  | T+  95\n",
      "[382/723] Writing tensor blk.42.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  95\n",
      "[383/723] Writing tensor blk.42.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  95\n",
      "[384/723] Writing tensor blk.42.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  95\n",
      "[385/723] Writing tensor blk.42.attn_output.weight              | size   8192 x   8192  | type F16  | T+  95\n",
      "[386/723] Writing tensor blk.42.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  97\n",
      "[387/723] Writing tensor blk.42.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  97\n",
      "[388/723] Writing tensor blk.42.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  97\n",
      "[389/723] Writing tensor blk.42.attn_norm.weight                | size   8192           | type F32  | T+  98\n",
      "[390/723] Writing tensor blk.42.ffn_norm.weight                 | size   8192           | type F32  | T+  98\n",
      "[391/723] Writing tensor blk.43.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  98\n",
      "[392/723] Writing tensor blk.43.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  98\n",
      "[393/723] Writing tensor blk.43.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  98\n",
      "[394/723] Writing tensor blk.43.attn_output.weight              | size   8192 x   8192  | type F16  | T+  98\n",
      "[395/723] Writing tensor blk.43.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  99\n",
      "[396/723] Writing tensor blk.43.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 100\n",
      "[397/723] Writing tensor blk.43.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 100\n",
      "[398/723] Writing tensor blk.43.attn_norm.weight                | size   8192           | type F32  | T+ 100\n",
      "[399/723] Writing tensor blk.43.ffn_norm.weight                 | size   8192           | type F32  | T+ 100\n",
      "[400/723] Writing tensor blk.44.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 100\n",
      "[401/723] Writing tensor blk.44.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 100\n",
      "[402/723] Writing tensor blk.44.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 100\n",
      "[403/723] Writing tensor blk.44.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 100\n",
      "[404/723] Writing tensor blk.44.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 102\n",
      "[405/723] Writing tensor blk.44.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 102\n",
      "[406/723] Writing tensor blk.44.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 102\n",
      "[407/723] Writing tensor blk.44.attn_norm.weight                | size   8192           | type F32  | T+ 102\n",
      "[408/723] Writing tensor blk.44.ffn_norm.weight                 | size   8192           | type F32  | T+ 102\n",
      "[409/723] Writing tensor blk.45.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 102\n",
      "[410/723] Writing tensor blk.45.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 102\n",
      "[411/723] Writing tensor blk.45.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 102\n",
      "[412/723] Writing tensor blk.45.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 102\n",
      "[413/723] Writing tensor blk.45.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 104\n",
      "[414/723] Writing tensor blk.45.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 104\n",
      "[415/723] Writing tensor blk.45.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 104\n",
      "[416/723] Writing tensor blk.45.attn_norm.weight                | size   8192           | type F32  | T+ 104\n",
      "[417/723] Writing tensor blk.45.ffn_norm.weight                 | size   8192           | type F32  | T+ 104\n",
      "[418/723] Writing tensor blk.46.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 104\n",
      "[419/723] Writing tensor blk.46.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 105\n",
      "[420/723] Writing tensor blk.46.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 105\n",
      "[421/723] Writing tensor blk.46.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 105\n",
      "[422/723] Writing tensor blk.46.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 106\n",
      "[423/723] Writing tensor blk.46.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 106\n",
      "[424/723] Writing tensor blk.46.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 107\n",
      "[425/723] Writing tensor blk.46.attn_norm.weight                | size   8192           | type F32  | T+ 107\n",
      "[426/723] Writing tensor blk.46.ffn_norm.weight                 | size   8192           | type F32  | T+ 107\n",
      "[427/723] Writing tensor blk.47.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 107\n",
      "[428/723] Writing tensor blk.47.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 107\n",
      "[429/723] Writing tensor blk.47.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 107\n",
      "[430/723] Writing tensor blk.47.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 107\n",
      "[431/723] Writing tensor blk.47.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 108\n",
      "[432/723] Writing tensor blk.47.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 109\n",
      "[433/723] Writing tensor blk.47.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 109\n",
      "[434/723] Writing tensor blk.47.attn_norm.weight                | size   8192           | type F32  | T+ 109\n",
      "[435/723] Writing tensor blk.47.ffn_norm.weight                 | size   8192           | type F32  | T+ 109\n",
      "[436/723] Writing tensor blk.48.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 109\n",
      "[437/723] Writing tensor blk.48.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 109\n",
      "[438/723] Writing tensor blk.48.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 109\n",
      "[439/723] Writing tensor blk.48.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 109\n",
      "[440/723] Writing tensor blk.48.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 111\n",
      "[441/723] Writing tensor blk.48.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 111\n",
      "[442/723] Writing tensor blk.48.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 111\n",
      "[443/723] Writing tensor blk.48.attn_norm.weight                | size   8192           | type F32  | T+ 111\n",
      "[444/723] Writing tensor blk.48.ffn_norm.weight                 | size   8192           | type F32  | T+ 111\n",
      "[445/723] Writing tensor blk.49.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 111\n",
      "[446/723] Writing tensor blk.49.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 111\n",
      "[447/723] Writing tensor blk.49.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 111\n",
      "[448/723] Writing tensor blk.49.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 111\n",
      "[449/723] Writing tensor blk.49.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 113\n",
      "[450/723] Writing tensor blk.49.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 113\n",
      "[451/723] Writing tensor blk.49.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 113\n",
      "[452/723] Writing tensor blk.49.attn_norm.weight                | size   8192           | type F32  | T+ 114\n",
      "[453/723] Writing tensor blk.49.ffn_norm.weight                 | size   8192           | type F32  | T+ 114\n",
      "[454/723] Writing tensor blk.50.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 114\n",
      "[455/723] Writing tensor blk.50.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 114\n",
      "[456/723] Writing tensor blk.50.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 114\n",
      "[457/723] Writing tensor blk.50.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 114\n",
      "[458/723] Writing tensor blk.50.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 115\n",
      "[459/723] Writing tensor blk.50.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 115\n",
      "[460/723] Writing tensor blk.50.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 116\n",
      "[461/723] Writing tensor blk.50.attn_norm.weight                | size   8192           | type F32  | T+ 116\n",
      "[462/723] Writing tensor blk.50.ffn_norm.weight                 | size   8192           | type F32  | T+ 116\n",
      "[463/723] Writing tensor blk.51.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 116\n",
      "[464/723] Writing tensor blk.51.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 116\n",
      "[465/723] Writing tensor blk.51.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 116\n",
      "[466/723] Writing tensor blk.51.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 116\n",
      "[467/723] Writing tensor blk.51.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 118\n",
      "[468/723] Writing tensor blk.51.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 118\n",
      "[469/723] Writing tensor blk.51.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 119\n",
      "[470/723] Writing tensor blk.51.attn_norm.weight                | size   8192           | type F32  | T+ 119\n",
      "[471/723] Writing tensor blk.51.ffn_norm.weight                 | size   8192           | type F32  | T+ 119\n",
      "[472/723] Writing tensor blk.52.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 119\n",
      "[473/723] Writing tensor blk.52.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 119\n",
      "[474/723] Writing tensor blk.52.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 119\n",
      "[475/723] Writing tensor blk.52.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 119\n",
      "[476/723] Writing tensor blk.52.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 121\n",
      "[477/723] Writing tensor blk.52.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 121\n",
      "[478/723] Writing tensor blk.52.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 122\n",
      "[479/723] Writing tensor blk.52.attn_norm.weight                | size   8192           | type F32  | T+ 122\n",
      "[480/723] Writing tensor blk.52.ffn_norm.weight                 | size   8192           | type F32  | T+ 122\n",
      "[481/723] Writing tensor blk.53.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 122\n",
      "[482/723] Writing tensor blk.53.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 122\n",
      "[483/723] Writing tensor blk.53.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 122\n",
      "[484/723] Writing tensor blk.53.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 122\n",
      "[485/723] Writing tensor blk.53.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 123\n",
      "[486/723] Writing tensor blk.53.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 124\n",
      "[487/723] Writing tensor blk.53.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 124\n",
      "[488/723] Writing tensor blk.53.attn_norm.weight                | size   8192           | type F32  | T+ 124\n",
      "[489/723] Writing tensor blk.53.ffn_norm.weight                 | size   8192           | type F32  | T+ 124\n",
      "[490/723] Writing tensor blk.54.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 124\n",
      "[491/723] Writing tensor blk.54.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 124\n",
      "[492/723] Writing tensor blk.54.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 124\n",
      "[493/723] Writing tensor blk.54.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 124\n",
      "[494/723] Writing tensor blk.54.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 126\n",
      "[495/723] Writing tensor blk.54.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 126\n",
      "[496/723] Writing tensor blk.54.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 126\n",
      "[497/723] Writing tensor blk.54.attn_norm.weight                | size   8192           | type F32  | T+ 126\n",
      "[498/723] Writing tensor blk.54.ffn_norm.weight                 | size   8192           | type F32  | T+ 126\n",
      "[499/723] Writing tensor blk.55.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 126\n",
      "[500/723] Writing tensor blk.55.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 126\n",
      "[501/723] Writing tensor blk.55.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 126\n",
      "[502/723] Writing tensor blk.55.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 126\n",
      "[503/723] Writing tensor blk.55.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 128\n",
      "[504/723] Writing tensor blk.55.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 128\n",
      "[505/723] Writing tensor blk.55.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 128\n",
      "[506/723] Writing tensor blk.55.attn_norm.weight                | size   8192           | type F32  | T+ 128\n",
      "[507/723] Writing tensor blk.55.ffn_norm.weight                 | size   8192           | type F32  | T+ 128\n",
      "[508/723] Writing tensor blk.56.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 128\n",
      "[509/723] Writing tensor blk.56.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 129\n",
      "[510/723] Writing tensor blk.56.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 129\n",
      "[511/723] Writing tensor blk.56.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 129\n",
      "[512/723] Writing tensor blk.56.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 130\n",
      "[513/723] Writing tensor blk.56.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 131\n",
      "[514/723] Writing tensor blk.56.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 131\n",
      "[515/723] Writing tensor blk.56.attn_norm.weight                | size   8192           | type F32  | T+ 131\n",
      "[516/723] Writing tensor blk.56.ffn_norm.weight                 | size   8192           | type F32  | T+ 131\n",
      "[517/723] Writing tensor blk.57.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 131\n",
      "[518/723] Writing tensor blk.57.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 131\n",
      "[519/723] Writing tensor blk.57.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 131\n",
      "[520/723] Writing tensor blk.57.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 131\n",
      "[521/723] Writing tensor blk.57.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 133\n",
      "[522/723] Writing tensor blk.57.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 133\n",
      "[523/723] Writing tensor blk.57.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 133\n",
      "[524/723] Writing tensor blk.57.attn_norm.weight                | size   8192           | type F32  | T+ 133\n",
      "[525/723] Writing tensor blk.57.ffn_norm.weight                 | size   8192           | type F32  | T+ 133\n",
      "[526/723] Writing tensor blk.58.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 133\n",
      "[527/723] Writing tensor blk.58.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 133\n",
      "[528/723] Writing tensor blk.58.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 133\n",
      "[529/723] Writing tensor blk.58.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 133\n",
      "[530/723] Writing tensor blk.58.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 135\n",
      "[531/723] Writing tensor blk.58.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 135\n",
      "[532/723] Writing tensor blk.58.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 136\n",
      "[533/723] Writing tensor blk.58.attn_norm.weight                | size   8192           | type F32  | T+ 136\n",
      "[534/723] Writing tensor blk.58.ffn_norm.weight                 | size   8192           | type F32  | T+ 136\n",
      "[535/723] Writing tensor blk.59.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 136\n",
      "[536/723] Writing tensor blk.59.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 136\n",
      "[537/723] Writing tensor blk.59.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 136\n",
      "[538/723] Writing tensor blk.59.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 136\n",
      "[539/723] Writing tensor blk.59.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 137\n",
      "[540/723] Writing tensor blk.59.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 138\n",
      "[541/723] Writing tensor blk.59.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 138\n",
      "[542/723] Writing tensor blk.59.attn_norm.weight                | size   8192           | type F32  | T+ 138\n",
      "[543/723] Writing tensor blk.59.ffn_norm.weight                 | size   8192           | type F32  | T+ 138\n",
      "[544/723] Writing tensor blk.60.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 138\n",
      "[545/723] Writing tensor blk.60.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 138\n",
      "[546/723] Writing tensor blk.60.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 138\n",
      "[547/723] Writing tensor blk.60.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 138\n",
      "[548/723] Writing tensor blk.60.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 140\n",
      "[549/723] Writing tensor blk.60.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 140\n",
      "[550/723] Writing tensor blk.60.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 140\n",
      "[551/723] Writing tensor blk.60.attn_norm.weight                | size   8192           | type F32  | T+ 140\n",
      "[552/723] Writing tensor blk.60.ffn_norm.weight                 | size   8192           | type F32  | T+ 140\n",
      "[553/723] Writing tensor blk.61.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 140\n",
      "[554/723] Writing tensor blk.61.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 140\n",
      "[555/723] Writing tensor blk.61.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 140\n",
      "[556/723] Writing tensor blk.61.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 140\n",
      "[557/723] Writing tensor blk.61.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 142\n",
      "[558/723] Writing tensor blk.61.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 142\n",
      "[559/723] Writing tensor blk.61.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 143\n",
      "[560/723] Writing tensor blk.61.attn_norm.weight                | size   8192           | type F32  | T+ 143\n",
      "[561/723] Writing tensor blk.61.ffn_norm.weight                 | size   8192           | type F32  | T+ 143\n",
      "[562/723] Writing tensor blk.62.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 143\n",
      "[563/723] Writing tensor blk.62.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 143\n",
      "[564/723] Writing tensor blk.62.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 143\n",
      "[565/723] Writing tensor blk.62.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 143\n",
      "[566/723] Writing tensor blk.62.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 144\n",
      "[567/723] Writing tensor blk.62.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 145\n",
      "[568/723] Writing tensor blk.62.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 145\n",
      "[569/723] Writing tensor blk.62.attn_norm.weight                | size   8192           | type F32  | T+ 145\n",
      "[570/723] Writing tensor blk.62.ffn_norm.weight                 | size   8192           | type F32  | T+ 145\n",
      "[571/723] Writing tensor blk.63.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 145\n",
      "[572/723] Writing tensor blk.63.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 145\n",
      "[573/723] Writing tensor blk.63.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 145\n",
      "[574/723] Writing tensor blk.63.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 145\n",
      "[575/723] Writing tensor blk.63.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 147\n",
      "[576/723] Writing tensor blk.63.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 147\n",
      "[577/723] Writing tensor blk.63.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 147\n",
      "[578/723] Writing tensor blk.63.attn_norm.weight                | size   8192           | type F32  | T+ 147\n",
      "[579/723] Writing tensor blk.63.ffn_norm.weight                 | size   8192           | type F32  | T+ 147\n",
      "[580/723] Writing tensor blk.64.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 147\n",
      "[581/723] Writing tensor blk.64.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 147\n",
      "[582/723] Writing tensor blk.64.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 147\n",
      "[583/723] Writing tensor blk.64.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 147\n",
      "[584/723] Writing tensor blk.64.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 149\n",
      "[585/723] Writing tensor blk.64.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 149\n",
      "[586/723] Writing tensor blk.64.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 149\n",
      "[587/723] Writing tensor blk.64.attn_norm.weight                | size   8192           | type F32  | T+ 149\n",
      "[588/723] Writing tensor blk.64.ffn_norm.weight                 | size   8192           | type F32  | T+ 149\n",
      "[589/723] Writing tensor blk.65.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 149\n",
      "[590/723] Writing tensor blk.65.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 149\n",
      "[591/723] Writing tensor blk.65.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 149\n",
      "[592/723] Writing tensor blk.65.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 149\n",
      "[593/723] Writing tensor blk.65.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 151\n",
      "[594/723] Writing tensor blk.65.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 151\n",
      "[595/723] Writing tensor blk.65.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 151\n",
      "[596/723] Writing tensor blk.65.attn_norm.weight                | size   8192           | type F32  | T+ 152\n",
      "[597/723] Writing tensor blk.65.ffn_norm.weight                 | size   8192           | type F32  | T+ 152\n",
      "[598/723] Writing tensor blk.66.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 152\n",
      "[599/723] Writing tensor blk.66.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 152\n",
      "[600/723] Writing tensor blk.66.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 152\n",
      "[601/723] Writing tensor blk.66.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 152\n",
      "[602/723] Writing tensor blk.66.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 153\n",
      "[603/723] Writing tensor blk.66.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 154\n",
      "[604/723] Writing tensor blk.66.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 154\n",
      "[605/723] Writing tensor blk.66.attn_norm.weight                | size   8192           | type F32  | T+ 154\n",
      "[606/723] Writing tensor blk.66.ffn_norm.weight                 | size   8192           | type F32  | T+ 154\n",
      "[607/723] Writing tensor blk.67.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 154\n",
      "[608/723] Writing tensor blk.67.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 154\n",
      "[609/723] Writing tensor blk.67.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 154\n",
      "[610/723] Writing tensor blk.67.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 154\n",
      "[611/723] Writing tensor blk.67.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 156\n",
      "[612/723] Writing tensor blk.67.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 156\n",
      "[613/723] Writing tensor blk.67.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 156\n",
      "[614/723] Writing tensor blk.67.attn_norm.weight                | size   8192           | type F32  | T+ 156\n",
      "[615/723] Writing tensor blk.67.ffn_norm.weight                 | size   8192           | type F32  | T+ 156\n",
      "[616/723] Writing tensor blk.68.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 156\n",
      "[617/723] Writing tensor blk.68.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 156\n",
      "[618/723] Writing tensor blk.68.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 156\n",
      "[619/723] Writing tensor blk.68.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 156\n",
      "[620/723] Writing tensor blk.68.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 158\n",
      "[621/723] Writing tensor blk.68.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 158\n",
      "[622/723] Writing tensor blk.68.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 158\n",
      "[623/723] Writing tensor blk.68.attn_norm.weight                | size   8192           | type F32  | T+ 159\n",
      "[624/723] Writing tensor blk.68.ffn_norm.weight                 | size   8192           | type F32  | T+ 159\n",
      "[625/723] Writing tensor blk.69.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 159\n",
      "[626/723] Writing tensor blk.69.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 159\n",
      "[627/723] Writing tensor blk.69.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 159\n",
      "[628/723] Writing tensor blk.69.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 159\n",
      "[629/723] Writing tensor blk.69.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 160\n",
      "[630/723] Writing tensor blk.69.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 161\n",
      "[631/723] Writing tensor blk.69.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 161\n",
      "[632/723] Writing tensor blk.69.attn_norm.weight                | size   8192           | type F32  | T+ 161\n",
      "[633/723] Writing tensor blk.69.ffn_norm.weight                 | size   8192           | type F32  | T+ 161\n",
      "[634/723] Writing tensor blk.70.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 161\n",
      "[635/723] Writing tensor blk.70.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 161\n",
      "[636/723] Writing tensor blk.70.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 161\n",
      "[637/723] Writing tensor blk.70.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 161\n",
      "[638/723] Writing tensor blk.70.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 163\n",
      "[639/723] Writing tensor blk.70.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 163\n",
      "[640/723] Writing tensor blk.70.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 163\n",
      "[641/723] Writing tensor blk.70.attn_norm.weight                | size   8192           | type F32  | T+ 163\n",
      "[642/723] Writing tensor blk.70.ffn_norm.weight                 | size   8192           | type F32  | T+ 163\n",
      "[643/723] Writing tensor blk.71.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 163\n",
      "[644/723] Writing tensor blk.71.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 163\n",
      "[645/723] Writing tensor blk.71.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 163\n",
      "[646/723] Writing tensor blk.71.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 163\n",
      "[647/723] Writing tensor blk.71.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 165\n",
      "[648/723] Writing tensor blk.71.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 165\n",
      "[649/723] Writing tensor blk.71.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 165\n",
      "[650/723] Writing tensor blk.71.attn_norm.weight                | size   8192           | type F32  | T+ 165\n",
      "[651/723] Writing tensor blk.71.ffn_norm.weight                 | size   8192           | type F32  | T+ 165\n",
      "[652/723] Writing tensor blk.72.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 165\n",
      "[653/723] Writing tensor blk.72.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 165\n",
      "[654/723] Writing tensor blk.72.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 165\n",
      "[655/723] Writing tensor blk.72.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 165\n",
      "[656/723] Writing tensor blk.72.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 167\n",
      "[657/723] Writing tensor blk.72.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 167\n",
      "[658/723] Writing tensor blk.72.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 167\n",
      "[659/723] Writing tensor blk.72.attn_norm.weight                | size   8192           | type F32  | T+ 168\n",
      "[660/723] Writing tensor blk.72.ffn_norm.weight                 | size   8192           | type F32  | T+ 168\n",
      "[661/723] Writing tensor blk.73.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 168\n",
      "[662/723] Writing tensor blk.73.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 168\n",
      "[663/723] Writing tensor blk.73.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 168\n",
      "[664/723] Writing tensor blk.73.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 168\n",
      "[665/723] Writing tensor blk.73.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 169\n",
      "[666/723] Writing tensor blk.73.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 169\n",
      "[667/723] Writing tensor blk.73.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 170\n",
      "[668/723] Writing tensor blk.73.attn_norm.weight                | size   8192           | type F32  | T+ 170\n",
      "[669/723] Writing tensor blk.73.ffn_norm.weight                 | size   8192           | type F32  | T+ 170\n",
      "[670/723] Writing tensor blk.74.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 170\n",
      "[671/723] Writing tensor blk.74.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 170\n",
      "[672/723] Writing tensor blk.74.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 170\n",
      "[673/723] Writing tensor blk.74.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 170\n",
      "[674/723] Writing tensor blk.74.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 172\n",
      "[675/723] Writing tensor blk.74.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 172\n",
      "[676/723] Writing tensor blk.74.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 172\n",
      "[677/723] Writing tensor blk.74.attn_norm.weight                | size   8192           | type F32  | T+ 172\n",
      "[678/723] Writing tensor blk.74.ffn_norm.weight                 | size   8192           | type F32  | T+ 172\n",
      "[679/723] Writing tensor blk.75.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 172\n",
      "[680/723] Writing tensor blk.75.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 172\n",
      "[681/723] Writing tensor blk.75.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 172\n",
      "[682/723] Writing tensor blk.75.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 172\n",
      "[683/723] Writing tensor blk.75.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 174\n",
      "[684/723] Writing tensor blk.75.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 174\n",
      "[685/723] Writing tensor blk.75.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 174\n",
      "[686/723] Writing tensor blk.75.attn_norm.weight                | size   8192           | type F32  | T+ 174\n",
      "[687/723] Writing tensor blk.75.ffn_norm.weight                 | size   8192           | type F32  | T+ 174\n",
      "[688/723] Writing tensor blk.76.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 174\n",
      "[689/723] Writing tensor blk.76.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 174\n",
      "[690/723] Writing tensor blk.76.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 174\n",
      "[691/723] Writing tensor blk.76.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 174\n",
      "[692/723] Writing tensor blk.76.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 176\n",
      "[693/723] Writing tensor blk.76.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 176\n",
      "[694/723] Writing tensor blk.76.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 176\n",
      "[695/723] Writing tensor blk.76.attn_norm.weight                | size   8192           | type F32  | T+ 177\n",
      "[696/723] Writing tensor blk.76.ffn_norm.weight                 | size   8192           | type F32  | T+ 177\n",
      "[697/723] Writing tensor blk.77.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 177\n",
      "[698/723] Writing tensor blk.77.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 177\n",
      "[699/723] Writing tensor blk.77.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 177\n",
      "[700/723] Writing tensor blk.77.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 177\n",
      "[701/723] Writing tensor blk.77.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 178\n",
      "[702/723] Writing tensor blk.77.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 179\n",
      "[703/723] Writing tensor blk.77.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 179\n",
      "[704/723] Writing tensor blk.77.attn_norm.weight                | size   8192           | type F32  | T+ 179\n",
      "[705/723] Writing tensor blk.77.ffn_norm.weight                 | size   8192           | type F32  | T+ 179\n",
      "[706/723] Writing tensor blk.78.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 179\n",
      "[707/723] Writing tensor blk.78.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 179\n",
      "[708/723] Writing tensor blk.78.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 179\n",
      "[709/723] Writing tensor blk.78.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 179\n",
      "[710/723] Writing tensor blk.78.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 181\n",
      "[711/723] Writing tensor blk.78.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 181\n",
      "[712/723] Writing tensor blk.78.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 181\n",
      "[713/723] Writing tensor blk.78.attn_norm.weight                | size   8192           | type F32  | T+ 181\n",
      "[714/723] Writing tensor blk.78.ffn_norm.weight                 | size   8192           | type F32  | T+ 181\n",
      "[715/723] Writing tensor blk.79.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 181\n",
      "[716/723] Writing tensor blk.79.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 181\n",
      "[717/723] Writing tensor blk.79.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 181\n",
      "[718/723] Writing tensor blk.79.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 181\n",
      "[719/723] Writing tensor blk.79.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 183\n",
      "[720/723] Writing tensor blk.79.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 183\n",
      "[721/723] Writing tensor blk.79.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 183\n",
      "[722/723] Writing tensor blk.79.attn_norm.weight                | size   8192           | type F32  | T+ 183\n",
      "[723/723] Writing tensor blk.79.ffn_norm.weight                 | size   8192           | type F32  | T+ 183\n",
      "Wrote models/70B-v2/ggml-model-f16.gguf\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n"
     ]
    }
   ],
   "source": [
    "# convert the models to ggml FP16 format\n",
    "!python3 convert.py models/7B-v2/\n",
    "!python3 convert.py models/13B-v2/\n",
    "!python3 convert.py models/70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "954d1eb9-d1d6-4525-8b0f-3b5809ad2d84",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I llama.cpp build info: \n",
      "I UNAME_S:   Linux\n",
      "I UNAME_P:   x86_64\n",
      "I UNAME_M:   x86_64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi\n",
      "I NVCCFLAGS:  \n",
      "I LDFLAGS:    \n",
      "I CC:        cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "I CXX:       g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "\n",
      "rm -vrf *.o tests/*.o *.so *.dll benchmark-matmult common/build-info.cpp *.dot *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead tests/test-c.o metal tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope tests/test-backend-ops\n",
      "removed 'build-info.o'\n",
      "removed 'common.o'\n",
      "removed 'console.o'\n",
      "removed 'ggml-alloc.o'\n",
      "removed 'ggml-backend.o'\n",
      "removed 'ggml-cuda.o'\n",
      "removed 'ggml-quants.o'\n",
      "removed 'ggml.o'\n",
      "removed 'grammar-parser.o'\n",
      "removed 'llama.o'\n",
      "removed 'sampling.o'\n",
      "removed 'train.o'\n",
      "removed 'tests/test-c.o'\n",
      "removed 'benchmark-matmult'\n",
      "removed 'common/build-info.cpp'\n",
      "removed 'main'\n",
      "removed 'quantize'\n",
      "removed 'quantize-stats'\n",
      "removed 'perplexity'\n",
      "removed 'embedding'\n",
      "removed 'vdot'\n",
      "removed 'q8dot'\n",
      "removed 'train-text-from-scratch'\n",
      "removed 'convert-llama2c-to-ggml'\n",
      "removed 'simple'\n",
      "removed 'batched'\n",
      "removed 'batched-bench'\n",
      "removed 'save-load-state'\n",
      "removed 'gguf'\n",
      "removed 'llama-bench'\n",
      "removed 'libllava.a'\n",
      "removed 'llava-cli'\n",
      "removed 'baby-llama'\n",
      "removed 'beam-search'\n",
      "removed 'speculative'\n",
      "removed 'infill'\n",
      "removed 'tokenize'\n",
      "removed 'parallel'\n",
      "removed 'finetune'\n",
      "removed 'export-lora'\n",
      "removed 'lookahead'\n",
      "I llama.cpp build info: \n",
      "I UNAME_S:   Linux\n",
      "I UNAME_P:   x86_64\n",
      "I UNAME_M:   x86_64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi\n",
      "I NVCCFLAGS: --forward-unknown-to-host-compiler -use_fast_math -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 \n",
      "I LDFLAGS:   -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "I CC:        cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "I CXX:       g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml.c -o ggml.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c llama.cpp -o llama.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/common.cpp -o common.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/sampling.cpp -o sampling.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/grammar-parser.cpp -o grammar-parser.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/console.cpp -o console.o\n",
      "nvcc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  --forward-unknown-to-host-compiler -use_fast_math -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128  -Wno-pedantic -Xcompiler \"-Wno-array-bounds -Wno-format-truncation -Wextra-semi\" -c ggml-cuda.cu -o ggml-cuda.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml-alloc.c -o ggml-alloc.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml-backend.c -o ggml-backend.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion     -c ggml-quants.c -o ggml-quants.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/train.cpp -o train.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion  -c tests/test-c.c -o tests/test-c.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/build-info.cpp -o build-info.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi pocs/vdot/vdot.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o vdot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi pocs/vdot/q8dot.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o q8dot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o benchmark-matmult -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/export-lora/export-lora.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o export-lora -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/main/main.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o main -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/quantize/quantize.cpp build-info.o ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize-stats -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/perplexity/perplexity.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o perplexity -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/embedding/embedding.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o embedding -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o train-text-from-scratch -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o convert-llama2c-to-ggml -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/simple/simple.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o simple -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/batched/batched.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o save-load-state -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iexamples/server examples/server/server.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o server -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib   -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/gguf/gguf.cpp ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o gguf -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/llama-bench/llama-bench.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llama-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -static -fPIC -c examples/llava/llava.cpp -o libllava.a -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/llava/llava-cli.cpp examples/llava/clip.cpp examples/llava/llava.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llava-cli -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib  -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o baby-llama -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/beam-search/beam-search.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o beam-search -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/speculative/speculative.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o speculative -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/infill/infill.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o infill -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/tokenize/tokenize.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o tokenize -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/parallel/parallel.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o parallel -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/finetune/finetune.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o finetune -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/lookahead/lookahead.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o lookahead -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
      "\n",
      "====  Run ./main -h for help.  ====\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# metal build\n",
    "!make clean && LLAMA_CUBLAS=1 make -j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c99bdabe-ce05-4e4a-bb7f-1ad00b66e57e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/7B-v2/ggml-model-f16.gguf' to './models/7B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llama_model_quantize_internal: meta size = 1714336 bytes\n",
      "[   1/ 291]                    token_embd.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   250.00 MiB ->    70.31 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   2/ 291]                   output_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   3/ 291]                        output.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   250.00 MiB ->   102.54 MiB | hist: \n",
      "[   4/ 291]                  blk.0.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.034 0.008 0.012 0.019 0.031 0.050 0.084 0.149 0.256 0.150 0.084 0.051 0.031 0.019 0.012 0.010 \n",
      "[   5/ 291]                  blk.0.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.034 0.008 0.013 0.021 0.033 0.054 0.089 0.150 0.226 0.151 0.089 0.054 0.033 0.021 0.013 0.011 \n",
      "[   6/ 291]                  blk.0.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.036 0.053 0.074 0.096 0.117 0.129 0.117 0.096 0.074 0.053 0.036 0.024 0.020 \n",
      "[   7/ 291]             blk.0.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.035 0.011 0.017 0.028 0.044 0.068 0.100 0.135 0.155 0.135 0.100 0.068 0.044 0.028 0.017 0.014 \n",
      "[   8/ 291]                blk.0.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   9/ 291]                blk.0.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  10/ 291]                  blk.0.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  11/ 291]               blk.0.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  12/ 291]                blk.0.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  13/ 291]                  blk.1.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.013 0.022 0.034 0.052 0.074 0.098 0.121 0.132 0.121 0.098 0.074 0.052 0.034 0.022 0.018 \n",
      "[  14/ 291]                  blk.1.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.013 0.022 0.034 0.051 0.074 0.099 0.121 0.132 0.121 0.099 0.074 0.051 0.034 0.022 0.018 \n",
      "[  15/ 291]                  blk.1.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.014 0.023 0.035 0.052 0.073 0.097 0.119 0.130 0.119 0.097 0.074 0.052 0.035 0.023 0.019 \n",
      "[  16/ 291]             blk.1.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.035 0.012 0.020 0.031 0.047 0.070 0.098 0.129 0.146 0.129 0.099 0.070 0.047 0.031 0.020 0.016 \n",
      "[  17/ 291]                blk.1.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 291]                blk.1.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 291]                  blk.1.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 291]               blk.1.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  21/ 291]                blk.1.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  22/ 291]                  blk.2.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.096 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  23/ 291]                  blk.2.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.124 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[  24/ 291]                  blk.2.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.120 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  25/ 291]             blk.2.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  26/ 291]                blk.2.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 291]                blk.2.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 291]                  blk.2.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 291]               blk.2.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  30/ 291]                blk.2.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  31/ 291]                  blk.3.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  32/ 291]                  blk.3.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  33/ 291]                  blk.3.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  34/ 291]             blk.3.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  35/ 291]                blk.3.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 291]                blk.3.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  37/ 291]                  blk.3.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 291]               blk.3.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  39/ 291]                blk.3.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  40/ 291]                  blk.4.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[  41/ 291]                  blk.4.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  42/ 291]                  blk.4.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  43/ 291]             blk.4.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  44/ 291]                blk.4.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 291]                blk.4.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  46/ 291]                  blk.4.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  47/ 291]               blk.4.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  48/ 291]                blk.4.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  49/ 291]                  blk.5.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  50/ 291]                  blk.5.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 291]                  blk.5.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  52/ 291]             blk.5.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  53/ 291]                blk.5.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 291]                blk.5.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  55/ 291]                  blk.5.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  56/ 291]               blk.5.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  57/ 291]                blk.5.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  58/ 291]                  blk.6.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  59/ 291]                  blk.6.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  60/ 291]                  blk.6.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  61/ 291]             blk.6.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  62/ 291]                blk.6.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 291]                blk.6.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 291]                  blk.6.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 291]               blk.6.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  66/ 291]                blk.6.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  67/ 291]                  blk.7.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  68/ 291]                  blk.7.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  69/ 291]                  blk.7.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  70/ 291]             blk.7.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  71/ 291]                blk.7.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 291]                blk.7.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  73/ 291]                  blk.7.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 291]               blk.7.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  75/ 291]                blk.7.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  76/ 291]                  blk.8.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 291]                  blk.8.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 291]                  blk.8.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  79/ 291]             blk.8.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 291]                blk.8.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 291]                blk.8.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  82/ 291]                  blk.8.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  83/ 291]               blk.8.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  84/ 291]                blk.8.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  85/ 291]                  blk.9.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 291]                  blk.9.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  87/ 291]                  blk.9.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  88/ 291]             blk.9.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  89/ 291]                blk.9.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 291]                blk.9.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  91/ 291]                  blk.9.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  92/ 291]               blk.9.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  93/ 291]                blk.9.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  94/ 291]                 blk.10.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  95/ 291]                 blk.10.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  96/ 291]                 blk.10.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  97/ 291]            blk.10.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  98/ 291]               blk.10.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 291]               blk.10.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 100/ 291]                 blk.10.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 101/ 291]              blk.10.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 102/ 291]               blk.10.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 103/ 291]                 blk.11.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 291]                 blk.11.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 291]                 blk.11.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 106/ 291]            blk.11.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 107/ 291]               blk.11.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 291]               blk.11.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 109/ 291]                 blk.11.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 110/ 291]              blk.11.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 111/ 291]               blk.11.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 112/ 291]                 blk.12.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 113/ 291]                 blk.12.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 291]                 blk.12.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 291]            blk.12.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 116/ 291]               blk.12.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 291]               blk.12.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 118/ 291]                 blk.12.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 291]              blk.12.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 120/ 291]               blk.12.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 121/ 291]                 blk.13.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 291]                 blk.13.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 123/ 291]                 blk.13.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 291]            blk.13.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 125/ 291]               blk.13.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 291]               blk.13.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 127/ 291]                 blk.13.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 291]              blk.13.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 129/ 291]               blk.13.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 130/ 291]                 blk.14.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 131/ 291]                 blk.14.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 291]                 blk.14.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 291]            blk.14.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 134/ 291]               blk.14.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 291]               blk.14.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 136/ 291]                 blk.14.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 291]              blk.14.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 138/ 291]               blk.14.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 139/ 291]                 blk.15.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 291]                 blk.15.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 141/ 291]                 blk.15.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 291]            blk.15.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 143/ 291]               blk.15.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 291]               blk.15.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 145/ 291]                 blk.15.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 291]              blk.15.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 147/ 291]               blk.15.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 148/ 291]                 blk.16.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 291]                 blk.16.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 150/ 291]                 blk.16.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 291]            blk.16.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 152/ 291]               blk.16.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 291]               blk.16.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 154/ 291]                 blk.16.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 291]              blk.16.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 156/ 291]               blk.16.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 157/ 291]                 blk.17.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 291]                 blk.17.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 159/ 291]                 blk.17.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 291]            blk.17.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 161/ 291]               blk.17.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 291]               blk.17.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 163/ 291]                 blk.17.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 291]              blk.17.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 165/ 291]               blk.17.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 166/ 291]                 blk.18.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 167/ 291]                 blk.18.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 168/ 291]                 blk.18.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 291]            blk.18.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 170/ 291]               blk.18.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 291]               blk.18.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 172/ 291]                 blk.18.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 291]              blk.18.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 174/ 291]               blk.18.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 175/ 291]                 blk.19.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 176/ 291]                 blk.19.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 177/ 291]                 blk.19.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 291]            blk.19.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 179/ 291]               blk.19.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 291]               blk.19.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 291]                 blk.19.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 291]              blk.19.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 183/ 291]               blk.19.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 184/ 291]                 blk.20.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 291]                 blk.20.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 186/ 291]                 blk.20.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 291]            blk.20.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 188/ 291]               blk.20.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 291]               blk.20.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 291]                 blk.20.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 291]              blk.20.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 192/ 291]               blk.20.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 193/ 291]                 blk.21.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 194/ 291]                 blk.21.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 195/ 291]                 blk.21.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 291]            blk.21.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 291]               blk.21.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 291]               blk.21.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 291]                 blk.21.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 291]              blk.21.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 201/ 291]               blk.21.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 202/ 291]                 blk.22.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 203/ 291]                 blk.22.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 204/ 291]                 blk.22.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 205/ 291]            blk.22.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 291]               blk.22.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 291]               blk.22.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 291]                 blk.22.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 291]              blk.22.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 210/ 291]               blk.22.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 211/ 291]                 blk.23.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 212/ 291]                 blk.23.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 213/ 291]                 blk.23.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 214/ 291]            blk.23.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 215/ 291]               blk.23.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 291]               blk.23.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 291]                 blk.23.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 291]              blk.23.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 219/ 291]               blk.23.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 220/ 291]                 blk.24.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 221/ 291]                 blk.24.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 222/ 291]                 blk.24.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 291]            blk.24.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 224/ 291]               blk.24.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 291]               blk.24.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 291]                 blk.24.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 291]              blk.24.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 228/ 291]               blk.24.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 229/ 291]                 blk.25.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 291]                 blk.25.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 231/ 291]                 blk.25.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 291]            blk.25.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 233/ 291]               blk.25.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 291]               blk.25.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 291]                 blk.25.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 291]              blk.25.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 237/ 291]               blk.25.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 238/ 291]                 blk.26.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 239/ 291]                 blk.26.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 240/ 291]                 blk.26.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 241/ 291]            blk.26.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 242/ 291]               blk.26.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 291]               blk.26.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 291]                 blk.26.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 291]              blk.26.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 246/ 291]               blk.26.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 247/ 291]                 blk.27.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 248/ 291]                 blk.27.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 249/ 291]                 blk.27.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 291]            blk.27.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 291]               blk.27.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 291]               blk.27.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 291]                 blk.27.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 291]              blk.27.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 255/ 291]               blk.27.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 256/ 291]                 blk.28.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 257/ 291]                 blk.28.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 258/ 291]                 blk.28.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 291]            blk.28.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 260/ 291]               blk.28.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 291]               blk.28.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 262/ 291]                 blk.28.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 291]              blk.28.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 264/ 291]               blk.28.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 265/ 291]                 blk.29.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 291]                 blk.29.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 267/ 291]                 blk.29.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 291]            blk.29.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 269/ 291]               blk.29.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 291]               blk.29.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 271/ 291]                 blk.29.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 291]              blk.29.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 273/ 291]               blk.29.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 274/ 291]                 blk.30.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 275/ 291]                 blk.30.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 276/ 291]                 blk.30.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 277/ 291]            blk.30.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 291]               blk.30.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 291]               blk.30.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 280/ 291]                 blk.30.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 291]              blk.30.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 282/ 291]               blk.30.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 283/ 291]                 blk.31.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 284/ 291]                 blk.31.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 285/ 291]                 blk.31.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 286/ 291]            blk.31.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 287/ 291]               blk.31.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 291]               blk.31.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[ 289/ 291]                 blk.31.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 290/ 291]              blk.31.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 291/ 291]               blk.31.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "llama_model_quantize_internal: model size  = 12853.02 MB\n",
      "llama_model_quantize_internal: quant size  =  3647.87 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 17181.91 ms\n",
      "main:    total time = 17181.91 ms\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/13B-v2/ggml-model-f16.gguf' to './models/13B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llama_model_quantize_internal: meta size = 1718656 bytes\n",
      "[   1/ 363]                    token_embd.weight - [ 5120, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   312.50 MiB ->    87.89 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   2/ 363]                   output_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[   3/ 363]                        output.weight - [ 5120, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   312.50 MiB ->   128.17 MiB | hist: \n",
      "[   4/ 363]                  blk.0.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.033 0.006 0.009 0.015 0.024 0.041 0.074 0.153 0.317 0.153 0.075 0.041 0.024 0.015 0.009 0.008 \n",
      "[   5/ 363]                  blk.0.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.033 0.006 0.010 0.015 0.025 0.043 0.078 0.158 0.293 0.158 0.078 0.043 0.025 0.015 0.010 0.008 \n",
      "[   6/ 363]                  blk.0.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.014 0.023 0.035 0.053 0.074 0.097 0.118 0.129 0.119 0.098 0.074 0.053 0.035 0.023 0.019 \n",
      "[   7/ 363]             blk.0.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.035 0.012 0.020 0.031 0.048 0.071 0.099 0.127 0.142 0.127 0.099 0.071 0.048 0.031 0.020 0.016 \n",
      "[   8/ 363]                blk.0.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[   9/ 363]                blk.0.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  10/ 363]                  blk.0.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  11/ 363]               blk.0.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  12/ 363]                blk.0.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  13/ 363]                  blk.1.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.072 0.098 0.124 0.139 0.124 0.098 0.072 0.050 0.033 0.021 0.017 \n",
      "[  14/ 363]                  blk.1.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.020 0.032 0.049 0.072 0.099 0.125 0.139 0.126 0.099 0.072 0.049 0.032 0.020 0.017 \n",
      "[  15/ 363]                  blk.1.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.075 0.097 0.116 0.124 0.116 0.097 0.075 0.054 0.037 0.024 0.020 \n",
      "[  16/ 363]             blk.1.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.021 0.033 0.051 0.073 0.099 0.123 0.134 0.123 0.099 0.073 0.051 0.034 0.021 0.018 \n",
      "[  17/ 363]                blk.1.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 363]                blk.1.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 363]                  blk.1.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 363]               blk.1.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  21/ 363]                blk.1.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  22/ 363]                  blk.2.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.123 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  23/ 363]                  blk.2.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.124 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[  24/ 363]                  blk.2.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  25/ 363]             blk.2.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.024 0.020 \n",
      "[  26/ 363]                blk.2.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 363]                blk.2.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 363]                  blk.2.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 363]               blk.2.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  30/ 363]                blk.2.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  31/ 363]                  blk.3.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.096 0.113 0.121 0.113 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[  32/ 363]                  blk.3.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[  33/ 363]                  blk.3.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  34/ 363]             blk.3.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  35/ 363]                blk.3.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 363]                blk.3.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  37/ 363]                  blk.3.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 363]               blk.3.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  39/ 363]                blk.3.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  40/ 363]                  blk.4.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  41/ 363]                  blk.4.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  42/ 363]                  blk.4.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[  43/ 363]             blk.4.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  44/ 363]                blk.4.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 363]                blk.4.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  46/ 363]                  blk.4.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  47/ 363]               blk.4.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  48/ 363]                blk.4.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  49/ 363]                  blk.5.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  50/ 363]                  blk.5.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 363]                  blk.5.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  52/ 363]             blk.5.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  53/ 363]                blk.5.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 363]                blk.5.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  55/ 363]                  blk.5.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  56/ 363]               blk.5.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  57/ 363]                blk.5.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  58/ 363]                  blk.6.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  59/ 363]                  blk.6.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  60/ 363]                  blk.6.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  61/ 363]             blk.6.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  62/ 363]                blk.6.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 363]                blk.6.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 363]                  blk.6.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 363]               blk.6.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  66/ 363]                blk.6.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  67/ 363]                  blk.7.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  68/ 363]                  blk.7.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  69/ 363]                  blk.7.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  70/ 363]             blk.7.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  71/ 363]                blk.7.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 363]                blk.7.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  73/ 363]                  blk.7.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 363]               blk.7.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  75/ 363]                blk.7.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  76/ 363]                  blk.8.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 363]                  blk.8.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 363]                  blk.8.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  79/ 363]             blk.8.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 363]                blk.8.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 363]                blk.8.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  82/ 363]                  blk.8.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  83/ 363]               blk.8.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  84/ 363]                blk.8.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  85/ 363]                  blk.9.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 363]                  blk.9.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  87/ 363]                  blk.9.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  88/ 363]             blk.9.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  89/ 363]                blk.9.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 363]                blk.9.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  91/ 363]                  blk.9.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  92/ 363]               blk.9.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  93/ 363]                blk.9.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  94/ 363]                 blk.10.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  95/ 363]                 blk.10.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  96/ 363]                 blk.10.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  97/ 363]            blk.10.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  98/ 363]               blk.10.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 363]               blk.10.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 100/ 363]                 blk.10.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 101/ 363]              blk.10.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 102/ 363]               blk.10.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 103/ 363]                 blk.11.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 363]                 blk.11.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 363]                 blk.11.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 106/ 363]            blk.11.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 107/ 363]               blk.11.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 363]               blk.11.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 109/ 363]                 blk.11.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 110/ 363]              blk.11.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 111/ 363]               blk.11.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 112/ 363]                 blk.12.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 113/ 363]                 blk.12.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 363]                 blk.12.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 363]            blk.12.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 116/ 363]               blk.12.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 363]               blk.12.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 118/ 363]                 blk.12.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 363]              blk.12.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 120/ 363]               blk.12.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 121/ 363]                 blk.13.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 363]                 blk.13.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 123/ 363]                 blk.13.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 363]            blk.13.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 125/ 363]               blk.13.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 363]               blk.13.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 127/ 363]                 blk.13.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 363]              blk.13.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 129/ 363]               blk.13.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 130/ 363]                 blk.14.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 131/ 363]                 blk.14.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 363]                 blk.14.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 363]            blk.14.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 134/ 363]               blk.14.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 363]               blk.14.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 136/ 363]                 blk.14.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 363]              blk.14.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 138/ 363]               blk.14.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 139/ 363]                 blk.15.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 363]                 blk.15.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 141/ 363]                 blk.15.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 363]            blk.15.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 143/ 363]               blk.15.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 363]               blk.15.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 145/ 363]                 blk.15.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 363]              blk.15.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 147/ 363]               blk.15.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 148/ 363]                 blk.16.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 363]                 blk.16.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 150/ 363]                 blk.16.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 363]            blk.16.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 152/ 363]               blk.16.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 363]               blk.16.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 154/ 363]                 blk.16.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 363]              blk.16.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 156/ 363]               blk.16.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 157/ 363]                 blk.17.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 363]                 blk.17.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 159/ 363]                 blk.17.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 363]            blk.17.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 161/ 363]               blk.17.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 363]               blk.17.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 163/ 363]                 blk.17.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 363]              blk.17.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 165/ 363]               blk.17.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 166/ 363]                 blk.18.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 167/ 363]                 blk.18.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 168/ 363]                 blk.18.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 363]            blk.18.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 170/ 363]               blk.18.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 363]               blk.18.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 172/ 363]                 blk.18.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 363]              blk.18.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 174/ 363]               blk.18.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 175/ 363]                 blk.19.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 176/ 363]                 blk.19.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 177/ 363]                 blk.19.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 363]            blk.19.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 179/ 363]               blk.19.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 363]               blk.19.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 363]                 blk.19.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 363]              blk.19.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 183/ 363]               blk.19.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 184/ 363]                 blk.20.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 363]                 blk.20.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 186/ 363]                 blk.20.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 363]            blk.20.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 188/ 363]               blk.20.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 363]               blk.20.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 363]                 blk.20.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 363]              blk.20.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 192/ 363]               blk.20.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 193/ 363]                 blk.21.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 194/ 363]                 blk.21.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 195/ 363]                 blk.21.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 363]            blk.21.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 363]               blk.21.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 363]               blk.21.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 363]                 blk.21.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 363]              blk.21.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 201/ 363]               blk.21.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 202/ 363]                 blk.22.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 203/ 363]                 blk.22.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 204/ 363]                 blk.22.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 205/ 363]            blk.22.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 363]               blk.22.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 363]               blk.22.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 363]                 blk.22.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 363]              blk.22.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 210/ 363]               blk.22.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 211/ 363]                 blk.23.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 212/ 363]                 blk.23.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 213/ 363]                 blk.23.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 214/ 363]            blk.23.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 215/ 363]               blk.23.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 363]               blk.23.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 363]                 blk.23.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 363]              blk.23.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 219/ 363]               blk.23.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 220/ 363]                 blk.24.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 221/ 363]                 blk.24.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 222/ 363]                 blk.24.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 363]            blk.24.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 224/ 363]               blk.24.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 363]               blk.24.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 363]                 blk.24.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 363]              blk.24.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 228/ 363]               blk.24.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 229/ 363]                 blk.25.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 363]                 blk.25.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 231/ 363]                 blk.25.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 363]            blk.25.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 233/ 363]               blk.25.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 363]               blk.25.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 363]                 blk.25.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 363]              blk.25.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 237/ 363]               blk.25.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 238/ 363]                 blk.26.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 239/ 363]                 blk.26.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 240/ 363]                 blk.26.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 241/ 363]            blk.26.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 242/ 363]               blk.26.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 363]               blk.26.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 363]                 blk.26.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 363]              blk.26.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 246/ 363]               blk.26.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 247/ 363]                 blk.27.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 248/ 363]                 blk.27.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 249/ 363]                 blk.27.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 363]            blk.27.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 363]               blk.27.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 363]               blk.27.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 363]                 blk.27.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 363]              blk.27.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 255/ 363]               blk.27.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 256/ 363]                 blk.28.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 257/ 363]                 blk.28.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 258/ 363]                 blk.28.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 363]            blk.28.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 260/ 363]               blk.28.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 363]               blk.28.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 262/ 363]                 blk.28.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 363]              blk.28.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 264/ 363]               blk.28.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 265/ 363]                 blk.29.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 363]                 blk.29.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 267/ 363]                 blk.29.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 363]            blk.29.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 269/ 363]               blk.29.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 363]               blk.29.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 271/ 363]                 blk.29.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 363]              blk.29.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 273/ 363]               blk.29.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 274/ 363]                 blk.30.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 275/ 363]                 blk.30.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 276/ 363]                 blk.30.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 277/ 363]            blk.30.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 363]               blk.30.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 363]               blk.30.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 280/ 363]                 blk.30.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 363]              blk.30.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 282/ 363]               blk.30.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 283/ 363]                 blk.31.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 284/ 363]                 blk.31.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 285/ 363]                 blk.31.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 286/ 363]            blk.31.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 287/ 363]               blk.31.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 363]               blk.31.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 289/ 363]                 blk.31.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 290/ 363]              blk.31.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 291/ 363]               blk.31.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 292/ 363]                 blk.32.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 293/ 363]                 blk.32.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 294/ 363]                 blk.32.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 295/ 363]            blk.32.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 296/ 363]               blk.32.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 297/ 363]               blk.32.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 298/ 363]                 blk.32.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 299/ 363]              blk.32.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 300/ 363]               blk.32.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 301/ 363]                 blk.33.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 302/ 363]                 blk.33.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 303/ 363]                 blk.33.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 304/ 363]            blk.33.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 305/ 363]               blk.33.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 306/ 363]               blk.33.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 307/ 363]                 blk.33.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 308/ 363]              blk.33.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 309/ 363]               blk.33.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 310/ 363]                 blk.34.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 311/ 363]                 blk.34.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 312/ 363]                 blk.34.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 313/ 363]            blk.34.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 314/ 363]               blk.34.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 315/ 363]               blk.34.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 316/ 363]                 blk.34.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 317/ 363]              blk.34.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 318/ 363]               blk.34.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 319/ 363]                 blk.35.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 320/ 363]                 blk.35.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 321/ 363]                 blk.35.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 322/ 363]            blk.35.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 323/ 363]               blk.35.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 324/ 363]               blk.35.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 325/ 363]                 blk.35.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 326/ 363]              blk.35.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 327/ 363]               blk.35.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 328/ 363]                 blk.36.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 329/ 363]                 blk.36.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 330/ 363]                 blk.36.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 331/ 363]            blk.36.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 332/ 363]               blk.36.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 333/ 363]               blk.36.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 334/ 363]                 blk.36.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 335/ 363]              blk.36.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 336/ 363]               blk.36.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 337/ 363]                 blk.37.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 338/ 363]                 blk.37.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 339/ 363]                 blk.37.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 340/ 363]            blk.37.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 341/ 363]               blk.37.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 342/ 363]               blk.37.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 343/ 363]                 blk.37.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 344/ 363]              blk.37.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 345/ 363]               blk.37.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 346/ 363]                 blk.38.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 347/ 363]                 blk.38.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 348/ 363]                 blk.38.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 349/ 363]            blk.38.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 350/ 363]               blk.38.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 351/ 363]               blk.38.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 352/ 363]                 blk.38.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 353/ 363]              blk.38.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 354/ 363]               blk.38.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 355/ 363]                 blk.39.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 356/ 363]                 blk.39.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 357/ 363]                 blk.39.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.120 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 358/ 363]            blk.39.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.096 0.113 0.121 0.113 0.096 0.076 0.055 0.038 0.025 0.021 \n",
      "[ 359/ 363]               blk.39.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 360/ 363]               blk.39.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.076 0.098 0.115 0.122 0.115 0.098 0.076 0.054 0.037 0.024 0.020 \n",
      "[ 361/ 363]                 blk.39.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 362/ 363]              blk.39.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 363/ 363]               blk.39.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "llama_model_quantize_internal: model size  = 24826.58 MB\n",
      "llama_model_quantize_internal: quant size  =  7023.90 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 28706.33 ms\n",
      "main:    total time = 28706.33 ms\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/70B-v2/ggml-model-f16.gguf' to './models/70B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type  f16:  562 tensors\n",
      "llama_model_quantize_internal: meta size = 1740160 bytes\n",
      "[   1/ 723]                    token_embd.weight - [ 8192, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   500.00 MiB ->   140.62 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[   2/ 723]                   output_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[   3/ 723]                        output.weight - [ 8192, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   500.00 MiB ->   205.08 MiB | hist: \n",
      "[   4/ 723]                  blk.0.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.034 0.009 0.014 0.023 0.037 0.059 0.093 0.147 0.198 0.148 0.093 0.059 0.037 0.023 0.014 0.012 \n",
      "[   5/ 723]                  blk.0.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.034 0.008 0.013 0.021 0.035 0.057 0.094 0.153 0.201 0.153 0.094 0.057 0.035 0.021 0.013 0.011 \n",
      "[   6/ 723]                  blk.0.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.096 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[   7/ 723]             blk.0.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.022 0.034 0.052 0.074 0.099 0.120 0.128 0.120 0.099 0.075 0.052 0.035 0.022 0.018 \n",
      "[   8/ 723]                blk.0.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   9/ 723]                blk.0.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.075 0.098 0.117 0.125 0.117 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  10/ 723]                  blk.0.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  11/ 723]               blk.0.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  12/ 723]                blk.0.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  13/ 723]                  blk.1.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.035 0.011 0.017 0.028 0.043 0.066 0.099 0.137 0.160 0.137 0.099 0.066 0.043 0.028 0.017 0.015 \n",
      "[  14/ 723]                  blk.1.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.073 0.099 0.124 0.135 0.124 0.099 0.073 0.050 0.033 0.021 0.018 \n",
      "[  15/ 723]                  blk.1.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.022 0.033 0.050 0.071 0.097 0.124 0.137 0.124 0.097 0.071 0.050 0.033 0.022 0.018 \n",
      "[  16/ 723]             blk.1.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.117 0.098 0.076 0.054 0.036 0.023 0.019 \n",
      "[  17/ 723]                blk.1.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 723]                blk.1.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 723]                  blk.1.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 723]               blk.1.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  21/ 723]                blk.1.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  22/ 723]                  blk.2.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.022 0.035 0.052 0.075 0.099 0.119 0.127 0.119 0.099 0.075 0.053 0.035 0.022 0.018 \n",
      "[  23/ 723]                  blk.2.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.117 0.125 0.117 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  24/ 723]                  blk.2.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  25/ 723]             blk.2.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  26/ 723]                blk.2.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 723]                blk.2.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 723]                  blk.2.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 723]               blk.2.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  30/ 723]                blk.2.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  31/ 723]                  blk.3.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  32/ 723]                  blk.3.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  33/ 723]                  blk.3.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  34/ 723]             blk.3.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  35/ 723]                blk.3.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 723]                blk.3.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  37/ 723]                  blk.3.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 723]               blk.3.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  39/ 723]                blk.3.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  40/ 723]                  blk.4.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[  41/ 723]                  blk.4.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.024 0.020 \n",
      "[  42/ 723]                  blk.4.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.024 0.020 \n",
      "[  43/ 723]             blk.4.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  44/ 723]                blk.4.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 723]                blk.4.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  46/ 723]                  blk.4.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  47/ 723]               blk.4.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  48/ 723]                blk.4.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  49/ 723]                  blk.5.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  50/ 723]                  blk.5.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 723]                  blk.5.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  52/ 723]             blk.5.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  53/ 723]                blk.5.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 723]                blk.5.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  55/ 723]                  blk.5.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  56/ 723]               blk.5.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  57/ 723]                blk.5.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  58/ 723]                  blk.6.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  59/ 723]                  blk.6.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  60/ 723]                  blk.6.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[  61/ 723]             blk.6.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  62/ 723]                blk.6.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 723]                blk.6.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 723]                  blk.6.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 723]               blk.6.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  66/ 723]                blk.6.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  67/ 723]                  blk.7.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  68/ 723]                  blk.7.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  69/ 723]                  blk.7.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  70/ 723]             blk.7.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  71/ 723]                blk.7.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 723]                blk.7.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  73/ 723]                  blk.7.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 723]               blk.7.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  75/ 723]                blk.7.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  76/ 723]                  blk.8.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 723]                  blk.8.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 723]                  blk.8.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  79/ 723]             blk.8.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 723]                blk.8.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 723]                blk.8.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  82/ 723]                  blk.8.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  83/ 723]               blk.8.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  84/ 723]                blk.8.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  85/ 723]                  blk.9.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 723]                  blk.9.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  87/ 723]                  blk.9.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  88/ 723]             blk.9.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  89/ 723]                blk.9.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 723]                blk.9.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  91/ 723]                  blk.9.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  92/ 723]               blk.9.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  93/ 723]                blk.9.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  94/ 723]                 blk.10.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  95/ 723]                 blk.10.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  96/ 723]                 blk.10.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  97/ 723]            blk.10.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  98/ 723]               blk.10.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 723]               blk.10.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 100/ 723]                 blk.10.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 101/ 723]              blk.10.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 102/ 723]               blk.10.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 103/ 723]                 blk.11.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 723]                 blk.11.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 723]                 blk.11.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 106/ 723]            blk.11.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 107/ 723]               blk.11.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 723]               blk.11.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 109/ 723]                 blk.11.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 110/ 723]              blk.11.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 111/ 723]               blk.11.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 112/ 723]                 blk.12.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 113/ 723]                 blk.12.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 723]                 blk.12.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 723]            blk.12.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 116/ 723]               blk.12.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 723]               blk.12.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 118/ 723]                 blk.12.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 723]              blk.12.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 120/ 723]               blk.12.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 121/ 723]                 blk.13.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 723]                 blk.13.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 123/ 723]                 blk.13.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 723]            blk.13.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 125/ 723]               blk.13.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 723]               blk.13.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 127/ 723]                 blk.13.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 723]              blk.13.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 129/ 723]               blk.13.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 130/ 723]                 blk.14.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 131/ 723]                 blk.14.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 723]                 blk.14.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 723]            blk.14.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 134/ 723]               blk.14.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 723]               blk.14.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 136/ 723]                 blk.14.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 723]              blk.14.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 138/ 723]               blk.14.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 139/ 723]                 blk.15.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 723]                 blk.15.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 141/ 723]                 blk.15.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 723]            blk.15.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 143/ 723]               blk.15.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 723]               blk.15.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 145/ 723]                 blk.15.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 723]              blk.15.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 147/ 723]               blk.15.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 148/ 723]                 blk.16.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 723]                 blk.16.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 150/ 723]                 blk.16.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 723]            blk.16.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 152/ 723]               blk.16.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 723]               blk.16.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 154/ 723]                 blk.16.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 723]              blk.16.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 156/ 723]               blk.16.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 157/ 723]                 blk.17.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 723]                 blk.17.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 159/ 723]                 blk.17.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 723]            blk.17.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 161/ 723]               blk.17.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 723]               blk.17.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 163/ 723]                 blk.17.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 723]              blk.17.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 165/ 723]               blk.17.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 166/ 723]                 blk.18.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 167/ 723]                 blk.18.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 168/ 723]                 blk.18.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 723]            blk.18.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 170/ 723]               blk.18.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 723]               blk.18.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 172/ 723]                 blk.18.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 723]              blk.18.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 174/ 723]               blk.18.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 175/ 723]                 blk.19.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 176/ 723]                 blk.19.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 177/ 723]                 blk.19.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 723]            blk.19.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 179/ 723]               blk.19.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 723]               blk.19.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 723]                 blk.19.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 723]              blk.19.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 183/ 723]               blk.19.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 184/ 723]                 blk.20.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 723]                 blk.20.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 186/ 723]                 blk.20.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 723]            blk.20.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 188/ 723]               blk.20.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 723]               blk.20.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 723]                 blk.20.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 723]              blk.20.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 192/ 723]               blk.20.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 193/ 723]                 blk.21.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 194/ 723]                 blk.21.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 195/ 723]                 blk.21.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 723]            blk.21.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 723]               blk.21.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 723]               blk.21.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 723]                 blk.21.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 723]              blk.21.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 201/ 723]               blk.21.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 202/ 723]                 blk.22.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 203/ 723]                 blk.22.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 204/ 723]                 blk.22.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 205/ 723]            blk.22.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 723]               blk.22.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 723]               blk.22.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 723]                 blk.22.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 723]              blk.22.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 210/ 723]               blk.22.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 211/ 723]                 blk.23.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 212/ 723]                 blk.23.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 213/ 723]                 blk.23.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 214/ 723]            blk.23.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 215/ 723]               blk.23.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 723]               blk.23.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 723]                 blk.23.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 723]              blk.23.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 219/ 723]               blk.23.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 220/ 723]                 blk.24.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 221/ 723]                 blk.24.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 222/ 723]                 blk.24.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 723]            blk.24.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 224/ 723]               blk.24.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 723]               blk.24.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 723]                 blk.24.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 723]              blk.24.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 228/ 723]               blk.24.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 229/ 723]                 blk.25.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 723]                 blk.25.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 231/ 723]                 blk.25.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 723]            blk.25.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 233/ 723]               blk.25.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 723]               blk.25.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 723]                 blk.25.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 723]              blk.25.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 237/ 723]               blk.25.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 238/ 723]                 blk.26.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 239/ 723]                 blk.26.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 240/ 723]                 blk.26.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 241/ 723]            blk.26.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 242/ 723]               blk.26.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 723]               blk.26.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 723]                 blk.26.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 723]              blk.26.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 246/ 723]               blk.26.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 247/ 723]                 blk.27.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 248/ 723]                 blk.27.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 249/ 723]                 blk.27.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 723]            blk.27.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 723]               blk.27.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 723]               blk.27.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 723]                 blk.27.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 723]              blk.27.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 255/ 723]               blk.27.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 256/ 723]                 blk.28.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 257/ 723]                 blk.28.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 258/ 723]                 blk.28.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 723]            blk.28.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 260/ 723]               blk.28.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 723]               blk.28.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 262/ 723]                 blk.28.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 723]              blk.28.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 264/ 723]               blk.28.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 265/ 723]                 blk.29.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 723]                 blk.29.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 267/ 723]                 blk.29.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 723]            blk.29.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 269/ 723]               blk.29.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 723]               blk.29.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 271/ 723]                 blk.29.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 723]              blk.29.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 273/ 723]               blk.29.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 274/ 723]                 blk.30.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 275/ 723]                 blk.30.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 276/ 723]                 blk.30.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 277/ 723]            blk.30.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 723]               blk.30.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 723]               blk.30.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 280/ 723]                 blk.30.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 723]              blk.30.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 282/ 723]               blk.30.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 283/ 723]                 blk.31.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 284/ 723]                 blk.31.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 285/ 723]                 blk.31.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 286/ 723]            blk.31.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 287/ 723]               blk.31.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 723]               blk.31.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 289/ 723]                 blk.31.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 290/ 723]              blk.31.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 291/ 723]               blk.31.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 292/ 723]                 blk.32.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 293/ 723]                 blk.32.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 294/ 723]                 blk.32.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 295/ 723]            blk.32.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 296/ 723]               blk.32.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 297/ 723]               blk.32.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 298/ 723]                 blk.32.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 299/ 723]              blk.32.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 300/ 723]               blk.32.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 301/ 723]                 blk.33.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 302/ 723]                 blk.33.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 303/ 723]                 blk.33.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 304/ 723]            blk.33.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 305/ 723]               blk.33.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 306/ 723]               blk.33.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 307/ 723]                 blk.33.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 308/ 723]              blk.33.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 309/ 723]               blk.33.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 310/ 723]                 blk.34.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 311/ 723]                 blk.34.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 312/ 723]                 blk.34.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 313/ 723]            blk.34.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 314/ 723]               blk.34.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 315/ 723]               blk.34.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 316/ 723]                 blk.34.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 317/ 723]              blk.34.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 318/ 723]               blk.34.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 319/ 723]                 blk.35.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 320/ 723]                 blk.35.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 321/ 723]                 blk.35.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 322/ 723]            blk.35.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 323/ 723]               blk.35.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 324/ 723]               blk.35.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 325/ 723]                 blk.35.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 326/ 723]              blk.35.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 327/ 723]               blk.35.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 328/ 723]                 blk.36.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 329/ 723]                 blk.36.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 330/ 723]                 blk.36.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 331/ 723]            blk.36.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 332/ 723]               blk.36.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 333/ 723]               blk.36.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 334/ 723]                 blk.36.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 335/ 723]              blk.36.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 336/ 723]               blk.36.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 337/ 723]                 blk.37.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 338/ 723]                 blk.37.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 339/ 723]                 blk.37.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.076 0.057 0.039 0.025 0.021 \n",
      "[ 340/ 723]            blk.37.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 341/ 723]               blk.37.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 342/ 723]               blk.37.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 343/ 723]                 blk.37.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 344/ 723]              blk.37.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 345/ 723]               blk.37.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 346/ 723]                 blk.38.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 347/ 723]                 blk.38.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 348/ 723]                 blk.38.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 349/ 723]            blk.38.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 350/ 723]               blk.38.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 351/ 723]               blk.38.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 352/ 723]                 blk.38.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 353/ 723]              blk.38.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 354/ 723]               blk.38.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 355/ 723]                 blk.39.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 356/ 723]                 blk.39.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 357/ 723]                 blk.39.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 358/ 723]            blk.39.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 359/ 723]               blk.39.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 360/ 723]               blk.39.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 361/ 723]                 blk.39.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 362/ 723]              blk.39.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 363/ 723]               blk.39.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 364/ 723]                 blk.40.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 365/ 723]                 blk.40.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 366/ 723]                 blk.40.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 367/ 723]            blk.40.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 368/ 723]               blk.40.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 369/ 723]               blk.40.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 370/ 723]                 blk.40.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 371/ 723]              blk.40.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 372/ 723]               blk.40.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 373/ 723]                 blk.41.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 374/ 723]                 blk.41.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 375/ 723]                 blk.41.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 376/ 723]            blk.41.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 377/ 723]               blk.41.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 378/ 723]               blk.41.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 379/ 723]                 blk.41.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 380/ 723]              blk.41.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 381/ 723]               blk.41.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 382/ 723]                 blk.42.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 383/ 723]                 blk.42.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 384/ 723]                 blk.42.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 385/ 723]            blk.42.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 386/ 723]               blk.42.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 387/ 723]               blk.42.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 388/ 723]                 blk.42.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 389/ 723]              blk.42.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 390/ 723]               blk.42.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 391/ 723]                 blk.43.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 392/ 723]                 blk.43.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 393/ 723]                 blk.43.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 394/ 723]            blk.43.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 395/ 723]               blk.43.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 396/ 723]               blk.43.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 397/ 723]                 blk.43.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 398/ 723]              blk.43.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 399/ 723]               blk.43.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 400/ 723]                 blk.44.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 401/ 723]                 blk.44.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 402/ 723]                 blk.44.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 403/ 723]            blk.44.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 404/ 723]               blk.44.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 405/ 723]               blk.44.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 406/ 723]                 blk.44.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 407/ 723]              blk.44.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 408/ 723]               blk.44.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 409/ 723]                 blk.45.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 410/ 723]                 blk.45.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 411/ 723]                 blk.45.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 412/ 723]            blk.45.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 413/ 723]               blk.45.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 414/ 723]               blk.45.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 415/ 723]                 blk.45.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 416/ 723]              blk.45.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 417/ 723]               blk.45.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 418/ 723]                 blk.46.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 419/ 723]                 blk.46.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 420/ 723]                 blk.46.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 421/ 723]            blk.46.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 422/ 723]               blk.46.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 423/ 723]               blk.46.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 424/ 723]                 blk.46.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 425/ 723]              blk.46.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 426/ 723]               blk.46.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 427/ 723]                 blk.47.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 428/ 723]                 blk.47.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.039 0.025 0.020 \n",
      "[ 429/ 723]                 blk.47.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 430/ 723]            blk.47.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 431/ 723]               blk.47.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 432/ 723]               blk.47.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 433/ 723]                 blk.47.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 434/ 723]              blk.47.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 435/ 723]               blk.47.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 436/ 723]                 blk.48.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 437/ 723]                 blk.48.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[ 438/ 723]                 blk.48.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 439/ 723]            blk.48.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 440/ 723]               blk.48.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 441/ 723]               blk.48.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 442/ 723]                 blk.48.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 443/ 723]              blk.48.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 444/ 723]               blk.48.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 445/ 723]                 blk.49.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 446/ 723]                 blk.49.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 447/ 723]                 blk.49.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 448/ 723]            blk.49.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 449/ 723]               blk.49.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 450/ 723]               blk.49.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 451/ 723]                 blk.49.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 452/ 723]              blk.49.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 453/ 723]               blk.49.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 454/ 723]                 blk.50.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 455/ 723]                 blk.50.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.123 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 456/ 723]                 blk.50.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 457/ 723]            blk.50.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 458/ 723]               blk.50.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 459/ 723]               blk.50.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 460/ 723]                 blk.50.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 461/ 723]              blk.50.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 462/ 723]               blk.50.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 463/ 723]                 blk.51.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 464/ 723]                 blk.51.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 465/ 723]                 blk.51.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 466/ 723]            blk.51.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 467/ 723]               blk.51.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 468/ 723]               blk.51.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 469/ 723]                 blk.51.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 470/ 723]              blk.51.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 471/ 723]               blk.51.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 472/ 723]                 blk.52.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 473/ 723]                 blk.52.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 474/ 723]                 blk.52.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 475/ 723]            blk.52.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 476/ 723]               blk.52.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 477/ 723]               blk.52.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 478/ 723]                 blk.52.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 479/ 723]              blk.52.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 480/ 723]               blk.52.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 481/ 723]                 blk.53.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 482/ 723]                 blk.53.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 483/ 723]                 blk.53.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 484/ 723]            blk.53.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 485/ 723]               blk.53.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 486/ 723]               blk.53.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 487/ 723]                 blk.53.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 488/ 723]              blk.53.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 489/ 723]               blk.53.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 490/ 723]                 blk.54.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 491/ 723]                 blk.54.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 492/ 723]                 blk.54.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 493/ 723]            blk.54.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 494/ 723]               blk.54.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 495/ 723]               blk.54.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 496/ 723]                 blk.54.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 497/ 723]              blk.54.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 498/ 723]               blk.54.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 499/ 723]                 blk.55.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 500/ 723]                 blk.55.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 501/ 723]                 blk.55.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 502/ 723]            blk.55.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 503/ 723]               blk.55.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 504/ 723]               blk.55.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 505/ 723]                 blk.55.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 506/ 723]              blk.55.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 507/ 723]               blk.55.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 508/ 723]                 blk.56.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 509/ 723]                 blk.56.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 510/ 723]                 blk.56.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 511/ 723]            blk.56.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 512/ 723]               blk.56.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 513/ 723]               blk.56.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 514/ 723]                 blk.56.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 515/ 723]              blk.56.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 516/ 723]               blk.56.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 517/ 723]                 blk.57.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 518/ 723]                 blk.57.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 519/ 723]                 blk.57.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 520/ 723]            blk.57.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 521/ 723]               blk.57.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 522/ 723]               blk.57.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 523/ 723]                 blk.57.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 524/ 723]              blk.57.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 525/ 723]               blk.57.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 526/ 723]                 blk.58.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 527/ 723]                 blk.58.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.097 0.117 0.126 0.117 0.097 0.075 0.054 0.037 0.023 0.019 \n",
      "[ 528/ 723]                 blk.58.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 529/ 723]            blk.58.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 530/ 723]               blk.58.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 531/ 723]               blk.58.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 532/ 723]                 blk.58.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 533/ 723]              blk.58.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 534/ 723]               blk.58.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 535/ 723]                 blk.59.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 536/ 723]                 blk.59.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.037 0.054 0.075 0.097 0.116 0.125 0.116 0.097 0.075 0.054 0.037 0.024 0.019 \n",
      "[ 537/ 723]                 blk.59.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 538/ 723]            blk.59.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 539/ 723]               blk.59.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 540/ 723]               blk.59.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 541/ 723]                 blk.59.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 542/ 723]              blk.59.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 543/ 723]               blk.59.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 544/ 723]                 blk.60.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.075 0.097 0.115 0.123 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 545/ 723]                 blk.60.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.072 0.097 0.123 0.140 0.123 0.097 0.072 0.050 0.034 0.021 0.018 \n",
      "[ 546/ 723]                 blk.60.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 547/ 723]            blk.60.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 548/ 723]               blk.60.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 549/ 723]               blk.60.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 550/ 723]                 blk.60.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 551/ 723]              blk.60.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 552/ 723]               blk.60.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 553/ 723]                 blk.61.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 554/ 723]                 blk.61.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.118 0.129 0.118 0.097 0.074 0.053 0.036 0.023 0.019 \n",
      "[ 555/ 723]                 blk.61.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 556/ 723]            blk.61.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 557/ 723]               blk.61.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 558/ 723]               blk.61.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 559/ 723]                 blk.61.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 560/ 723]              blk.61.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 561/ 723]               blk.61.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 562/ 723]                 blk.62.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 563/ 723]                 blk.62.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.118 0.129 0.118 0.097 0.074 0.053 0.036 0.023 0.019 \n",
      "[ 564/ 723]                 blk.62.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 565/ 723]            blk.62.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 566/ 723]               blk.62.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 567/ 723]               blk.62.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 568/ 723]                 blk.62.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 569/ 723]              blk.62.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 570/ 723]               blk.62.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 571/ 723]                 blk.63.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 572/ 723]                 blk.63.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.035 0.052 0.074 0.097 0.119 0.132 0.119 0.097 0.074 0.053 0.035 0.023 0.019 \n",
      "[ 573/ 723]                 blk.63.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 574/ 723]            blk.63.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 575/ 723]               blk.63.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 576/ 723]               blk.63.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 577/ 723]                 blk.63.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 578/ 723]              blk.63.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 579/ 723]               blk.63.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 580/ 723]                 blk.64.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 581/ 723]                 blk.64.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.036 0.054 0.075 0.097 0.117 0.125 0.117 0.097 0.075 0.054 0.037 0.024 0.019 \n",
      "[ 582/ 723]                 blk.64.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 583/ 723]            blk.64.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 584/ 723]               blk.64.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 585/ 723]               blk.64.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 586/ 723]                 blk.64.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 587/ 723]              blk.64.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 588/ 723]               blk.64.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 589/ 723]                 blk.65.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.115 0.122 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 590/ 723]                 blk.65.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.022 0.034 0.051 0.072 0.097 0.122 0.138 0.122 0.097 0.072 0.051 0.034 0.022 0.018 \n",
      "[ 591/ 723]                 blk.65.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 592/ 723]            blk.65.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 593/ 723]               blk.65.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 594/ 723]               blk.65.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 595/ 723]                 blk.65.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 596/ 723]              blk.65.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 597/ 723]               blk.65.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 598/ 723]                 blk.66.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 599/ 723]                 blk.66.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.119 0.130 0.119 0.097 0.074 0.053 0.035 0.023 0.019 \n",
      "[ 600/ 723]                 blk.66.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 601/ 723]            blk.66.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 602/ 723]               blk.66.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 603/ 723]               blk.66.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 604/ 723]                 blk.66.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 605/ 723]              blk.66.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 606/ 723]               blk.66.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 607/ 723]                 blk.67.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.097 0.116 0.125 0.117 0.097 0.075 0.054 0.036 0.023 0.019 \n",
      "[ 608/ 723]                 blk.67.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.035 0.053 0.074 0.097 0.119 0.133 0.119 0.096 0.073 0.052 0.035 0.023 0.019 \n",
      "[ 609/ 723]                 blk.67.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 610/ 723]            blk.67.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 611/ 723]               blk.67.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 612/ 723]               blk.67.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 613/ 723]                 blk.67.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 614/ 723]              blk.67.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 615/ 723]               blk.67.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 616/ 723]                 blk.68.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 617/ 723]                 blk.68.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 618/ 723]                 blk.68.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 619/ 723]            blk.68.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 620/ 723]               blk.68.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 621/ 723]               blk.68.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 622/ 723]                 blk.68.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 623/ 723]              blk.68.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 624/ 723]               blk.68.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 625/ 723]                 blk.69.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 626/ 723]                 blk.69.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 627/ 723]                 blk.69.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 628/ 723]            blk.69.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 629/ 723]               blk.69.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 630/ 723]               blk.69.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 631/ 723]                 blk.69.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 632/ 723]              blk.69.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 633/ 723]               blk.69.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 634/ 723]                 blk.70.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 635/ 723]                 blk.70.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 636/ 723]                 blk.70.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 637/ 723]            blk.70.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 638/ 723]               blk.70.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 639/ 723]               blk.70.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 640/ 723]                 blk.70.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 641/ 723]              blk.70.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 642/ 723]               blk.70.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 643/ 723]                 blk.71.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 644/ 723]                 blk.71.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 645/ 723]                 blk.71.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 646/ 723]            blk.71.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 647/ 723]               blk.71.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 648/ 723]               blk.71.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 649/ 723]                 blk.71.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 650/ 723]              blk.71.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 651/ 723]               blk.71.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 652/ 723]                 blk.72.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 653/ 723]                 blk.72.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 654/ 723]                 blk.72.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 655/ 723]            blk.72.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 656/ 723]               blk.72.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 657/ 723]               blk.72.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 658/ 723]                 blk.72.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 659/ 723]              blk.72.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 660/ 723]               blk.72.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 661/ 723]                 blk.73.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 662/ 723]                 blk.73.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 663/ 723]                 blk.73.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 664/ 723]            blk.73.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 665/ 723]               blk.73.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 666/ 723]               blk.73.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 667/ 723]                 blk.73.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 668/ 723]              blk.73.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 669/ 723]               blk.73.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 670/ 723]                 blk.74.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 671/ 723]                 blk.74.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 672/ 723]                 blk.74.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 673/ 723]            blk.74.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 674/ 723]               blk.74.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 675/ 723]               blk.74.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 676/ 723]                 blk.74.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 677/ 723]              blk.74.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 678/ 723]               blk.74.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 679/ 723]                 blk.75.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 680/ 723]                 blk.75.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 681/ 723]                 blk.75.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 682/ 723]            blk.75.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 683/ 723]               blk.75.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 684/ 723]               blk.75.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 685/ 723]                 blk.75.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 686/ 723]              blk.75.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 687/ 723]               blk.75.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 688/ 723]                 blk.76.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 689/ 723]                 blk.76.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 690/ 723]                 blk.76.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 691/ 723]            blk.76.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 692/ 723]               blk.76.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 693/ 723]               blk.76.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 694/ 723]                 blk.76.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 695/ 723]              blk.76.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 696/ 723]               blk.76.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 697/ 723]                 blk.77.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 698/ 723]                 blk.77.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 699/ 723]                 blk.77.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 700/ 723]            blk.77.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.040 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.040 0.026 0.021 \n",
      "[ 701/ 723]               blk.77.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 702/ 723]               blk.77.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 703/ 723]                 blk.77.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 704/ 723]              blk.77.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 705/ 723]               blk.77.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 706/ 723]                 blk.78.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 707/ 723]                 blk.78.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 708/ 723]                 blk.78.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 709/ 723]            blk.78.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 710/ 723]               blk.78.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 711/ 723]               blk.78.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 712/ 723]                 blk.78.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 713/ 723]              blk.78.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 714/ 723]               blk.78.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 715/ 723]                 blk.79.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 716/ 723]                 blk.79.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 717/ 723]                 blk.79.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 718/ 723]            blk.79.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.040 0.026 0.021 \n",
      "[ 719/ 723]               blk.79.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 720/ 723]               blk.79.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 721/ 723]                 blk.79.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 722/ 723]              blk.79.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 723/ 723]               blk.79.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "llama_model_quantize_internal: model size  = 131565.03 MB\n",
      "llama_model_quantize_internal: quant size  = 37070.73 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 105771.41 ms\n",
      "main:    total time = 105771.41 ms\n"
     ]
    }
   ],
   "source": [
    "# quantize the model to 4-bits (using q4_0 method)\n",
    "!./quantize ./models/7B-v2/ggml-model-f16.gguf ./models/7B-v2/ggml-model-q4_0.gguf q4_0\n",
    "!./quantize ./models/13B-v2/ggml-model-f16.gguf ./models/13B-v2/ggml-model-q4_0.gguf q4_0\n",
    "!./quantize ./models/70B-v2/ggml-model-f16.gguf ./models/70B-v2/ggml-model-q4_0.gguf q4_0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "416ca561-de1a-4094-ae0b-fd71408d45e6",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f0aede9-2f19-41e1-bb4f-1a1d30a00156",
   "metadata": {},
   "source": [
    "### 7B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c50d2ab-fc82-4119-8ac3-38ead2b8fee8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206407\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.98 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   70.42 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 3577.55 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.06 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of that profound statesman, Doctor Ayschaak, she triumphantly rode a whale; and was all of a twitter to electrify the world with the news, in a voice so turgid with meaning that Rodney, with all his military stores, could scarcely have invented such a cadenced silence. Four-and-twenty hours would never pass away wearisomely: – every one must confess himself to be fully satisfied, that a state so happy is worth fighting for, in every ballad he can compose upon the subject. \n",
      "France was at peace with all the world; and her citizens seemed as if they had never loved their country but after the English fashion. The only man who thought differently from common was a certain Monsieur Saccardier: – who alone had had courage to stand up for truth, justice, virtue, honesty, industry, temperance, sobriety, mercy, modesty and religion. This blessed man was quite unknown in the great world he lived in; and perhaps no one would ever have heard his name if he had not been betrayed into publishing an immensely long novel with an immense number of immensely long chapters, which was all about himself and what everybody said of him. When this work appeared it did a prodigious deal to alter the manners of the age; because the only thing people read in those days were novels – and it showed them the folly of reading novels at all. In fact it would be impossible for me to mention Mr Saccardier as an example of virtue unless I had first been enabled by his writings to see that the world is not such a wicked place after all. For instance I do not recollect ever having met with such a good man in my life; and therefore I know this cannot be a common case, but one very particular to myself. I remember being deeply shocked when I first heard of this most excellent man; for he was the last person that could possibly have been expected by any body to turn out as he did: because everybody said, from their childhood upwards, that there was not such a wicked place in all Europe as France – where even virtue was regarded with contempt and treated with ridicule.\n",
      "When I first heard of Mr Saccardier (for though his writings are a masterpiece they have been overlooked by the literary world), it would be difficult for me to convey my indignation at finding him such a person: but if I were not ashamed of appearing as wicked as he is good, I could tell you of things that are too shocking. I am sure even the most virtuous people in England have not any idea of what I mean by the depravity of his mind, or how little it has in common with their own. I was at first almost as much terrified as delighted to discover this good man: for as he had been praised from his childhood upwards (though he had always been a most unpromising pupil), and everybody who knew him had said, that though the world were such a wicked place, there was not an individual of them all half so good, I felt myself in danger of being crushed under the load of his virtues. Indeed the idea of having to deal with a person like Mr Saccardier is so new and strange to me, that my feelings on first meeting him are perfectly unaccountable even to myself. But if it be asked what sort of a man he is, I should reply – He is the most amiable good man that ever lived, and at the same time an object of unutterable contempt and disgust to me: but still, in spite of everything that could offend or shock me, I can think of nobody on this earth whose society is so dear.\n",
      "I cannot pretend that he does not excite the utmost terror and horror in me; for how can I help being frightened when it appears that a person who is an object of contempt to everybody else, should nevertheless be such an object of affection to myself? However, though these are certainly the most painful feelings which my heart has ever experienced, there is some little alleviation of them in finding that I am not alone in the world in being thus tormented.\n",
      "The history of Mr Sinclair's life, if it were written by an unprejudiced hand, might probably be found to be very much like many other lives: but as a proof of my prepossessions against him, you have only to hear the story as told me by himself; which I will now repeat.\n",
      "Mr Sinclair was born in a small town about twenty miles from Bristol. His father kept an inn there, and died when Mr Sinclair was about twelve years old: his mother had been a weak woman, without either\n",
      "llama_print_timings:        load time =    2946.55 ms\n",
      "llama_print_timings:      sample time =     158.15 ms /  1024 runs   (    0.15 ms per token,  6474.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     287.17 ms /   494 tokens (    0.58 ms per token,  1720.22 tokens per second)\n",
      "llama_print_timings:        eval time =   15413.36 ms /  1023 runs   (   15.07 ms per token,    66.37 tokens per second)\n",
      "llama_print_timings:       total time =   16098.10 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3e237938-4375-406a-b329-48d6d2363aa9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206429\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.98 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   70.42 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 3577.55 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.06 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m \n",
      " Under the French revolutionary calendar such dates as the following, are very appropriately called for: — \"What gross calumny ever yet was current concerning Mr. HUME in Scotland?\" asked I, a day or two since, on a certain social site. That question being a part of my contribution to a discussion regarding the life and times of that illustrious man; and being also intended as a general query directed to all who are interested therein; has brought forth replies, as it were, from several different quarters. And here are some extracts from such replies: — \n",
      "\"It is difficult to find in the entire annals of history an instance more fraught with the saddest lessons for us all... (the) unbounded eloquence, and almost unlimited power of reasoning that he possessed...\" [1] \n",
      "\"What a strange, sad thing it was! Hume was one of the finest of human beings. And so many of his friends died young.\" [2] \"The life and character of David Hum... has been... well-nigh lost sight of in recent years,\" said an American author; [3] whilst an Englishman declared himself to have seen a copy of _Hume's Life_ , \"the only one I could lay hands on,\" [4] but added there that it was not easy for him to obtain. [5] \"I remember as if it were yesterday, when I first read about Hume's death...\" [6] \"How wonderful a mind he had!\" exclaimed another correspondent; [7] and \"He died too soon,\" said yet another contributor to the discussion: — \n",
      "\"A book was written about him (David HUME). It is entitled, _Hume's Life._ I have it in my possession. I am unable to locate any others. Perhaps, a library could locate copies for your readers.\" [8] \n",
      "\"There are some interesting papers on the man by James Hogg — including a detailed biography of the man with extracts from his own autobiographical writings...\" (an excerpt of which is printed in my book) \"I am not aware if there have been any other writers about him.\" [9] \n",
      "\"Hume was a friend of David BLACKLOCK, a famous writer and poet who had a library near that of Hume. They were on very friendly terms. Blacklock published an account of the life of Hume in his book...\" (an excerpt from which is printed also in my work). [10] \n",
      "\"His _Life_ was published as early as 1822 by William Creech. It contains much original information...\" and \"It is still a great rarity.\" [11] \n",
      "And so on: all these remarks are cited here more or less verbatim in order to show the general state of Hume-research, as I have experienced it.\n",
      "Of course there have been many more biographies, notices and references to David Hume that my colleagues have collected over the years: the one by George D. Willson is the best known; and the bibliography prepared by John Y. T. MACFARLANE in connection with his Hume-biography is a model of its kind (see _op. cit._ , pp. 762–8). But my personal impression, to which I have given some attention, was that a complete, or even nearly complete, biographical and scholarly study on David Hume has still not been published: perhaps because such work is deemed too large for the scope of a single book-edition; or because it is difficult for an author to make a living in writing books which are read only by specialists. I am therefore more than glad that the present study fills this gap, and is published as a new edition of Hume's _History_.\n",
      "But what really attracted my interest was not the subject-matter of David Hume's historical works per se; rather it was their form: namely Hume's original and at times bold attempt to adapt an ancient historiographical tradition for his own time. It seems to me that this particular aspect, so far ignored in the scholarly literature on David Hume, deserves our special attention; indeed it is precisely Hume's specific contribution which made his work of a historical kind possible in the first place. I therefore intend to stress this feature of Hume's historical enterprise here – without wishing in any way to denigrate the traditional themes he handled (the rise and fall of empires, etc.) or their importance for a comprehensive understanding of European history from 1735 until 1764. But I think it is important that we look at Hume's historical\n",
      "llama_print_timings:        load time =    5714.75 ms\n",
      "llama_print_timings:      sample time =     155.64 ms /  1024 runs   (    0.15 ms per token,  6579.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     291.26 ms /   494 tokens (    0.59 ms per token,  1696.08 tokens per second)\n",
      "llama_print_timings:        eval time =   15450.28 ms /  1023 runs   (   15.10 ms per token,    66.21 tokens per second)\n",
      "llama_print_timings:       total time =   16148.13 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb2e9a98-ed63-4b0b-b858-8a69c3cde67f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206461\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.98 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   70.42 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 3577.55 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.06 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under this state of things, for fifteen or sixteen years that monied man of the world, Mr. Micawber (may he live long and may you never hear of him) had gone diligently to work; constructing a national debt so immense that all the blades of grass plucked up in making it could scarcely have lain agaze on the green field for less than three years: during which time every child in France might have drawn, with a straw, enough hay to carry away under its arm. But Mr Micawber was unlucky enough to be born under another zodiac sign besides that of the lion; and he never got his point. \n",
      "And Britain, with all their witchcraft at no time hitherto shown itself either signs or wonders greater than in the days when the three children were kicked out of the window: and yet, said Sancho, I see clearly enough that if such a miracle as that was performed in our days it would be set down to legerdemain; and so they were by those who were then living. \n",
      "The year of Grace 1835 stared Meg Merrilies full in the face – a big, fat, rosy-cheeked, dimpled, chubby-chubb, double chin, squab nose, wide mouth, pendulous underlip and bouncing breasted woman. And she was in a white nightcap with curls to it! And the room she sat in was the exact copy of one which had been furnished by the builder's hands for her very own mother some ten years before. It was the same, and yet not so.\n",
      "The chubbly chubby woman on this day at any rate wasn't what people would now call 'mad'. She was only eccentric – a little bit strange and peculiar in her ways, but still sensible. A most extraordinary phenomenon had occurred in the world of men. The old order of things had been reversed. Men were no longer the ruling sex. Women now ruled the roost. It was they who wore the trousers now; not them wearing skirts and petticoats like men.\n",
      "They used to wear breeches, but now they wore drawers instead – and the very sight of those two little dangling bobtails made every man's blood boil with rage. Men were the slaves now of women; and what made them especially so was that these women were the least reasonable creatures ever known in this world or the next. They never, never did anything for reason – and if they had any, they had no use for it, like a man has his coat pocket with change in it.\n",
      "Women's rule of the roost was also not as free and easy as men's. Women were slaves to their emotions, just as men are to their reason; but unlike a slave who is forced to do work against his own will for one he loves, women had no love for any man living.\n",
      "They didn't even love themselves enough to look after themselves – which was why they used to be so poor and miserable before this terrible change had come over them. It was this that made the world of men now so terrible too, though it was because they were not slaves but free. And these women, who no longer needed food or lodging from their husbands, who didn't love themselves enough to do anything for their own comfort and ease in life – even if they had any reason to be comfortable – had turned into creatures who would only serve men if it suited them, which wasn't very often; but when they did serve, the service was always against their own will, like a slave working for his master.\n",
      "Now there was no free man alive in this world, nor any slave, but all were made as slaves and as freemen by that change women had made over themselves to serve men. And since the world was now of two kinds – as it had always been before – both men's and women's – everyone now used his or her kind of mind against others; while in that earlier age, when everyone lived according to one kind of mind only, no one used his own mind for that, but only for what he himself had made.\n",
      "Men were the most foolish of these creatures who could never make anything right, whether it was wrong or right, except if they had been taught by a wise person, like a man teaches a dog; and women were the most cruel and hateful of them all – though also the worst sufferer of all.\n",
      "Every woman was cruel to every other woman; and because she didn't know enough about reason to see how it could be done without pain or discomfort, even her own sex suffered; for if\n",
      "llama_print_timings:        load time =    5595.18 ms\n",
      "llama_print_timings:      sample time =     158.44 ms /  1024 runs   (    0.15 ms per token,  6462.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     287.75 ms /   494 tokens (    0.58 ms per token,  1716.80 tokens per second)\n",
      "llama_print_timings:        eval time =   15450.50 ms /  1023 runs   (   15.10 ms per token,    66.21 tokens per second)\n",
      "llama_print_timings:       total time =   16134.95 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52c40741-4720-463b-a699-a05ae9266ab2",
   "metadata": {},
   "source": [
    "### 7B f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e3683e44-8c9a-4f85-abfc-37c024375357",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206496\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.13 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  250.11 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 12603.02 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors she entertained herself, while lying quietly in bed after dinner, by counting her afflictions, (of which she had many,) and exclaiming Gone! She did not know that these two words cost a pair of shoes to utter them; for it is hard to dress and undress on an average under five shillings. \n",
      " The Sun-fire steed of the day was Count Rumford's essay \"On the Declination of Credit\" (to which he had given the title of An Essay on the Scarcity of Gold and Silver in England.) It set off with a splash, in consequence of some powerful springs being got under it; but it soon fell into a gentle but constant gliding motion, so that some people thought they saw it go by. \n",
      " This piece of writing had a very quiet kind of popularity. To say truth, Mr. Rumford's name did not appear to great advantage in the early part of it (as being in a manner coeval with that of the writer of the pamphlet against Tithes) and there was something too novelty about his system, as it was called. It was started, however, by three or four great houses in London; where it met with very general support from the landed interest; but the trade was so much alarmed at first that many of them did not join in giving it a fair trial for several months; and when they began to give it a fair trial, the work had been going on some time already. At this time Mr. Pitt was Minister: which may be regarded as an advantage (in one respect) to the Bank Restriction Act of 1797. For that gentleman's public conduct on many points seemed to justify some people in the opinion that he did not altogether like paper-money; and if it had happened that he was Minister at this time, the measure might probably have been delayed until his successor should be chosen. However, it was carried through by a majority of one only, upon an address from the Houses of Lords and Commons praying His Majesty that he would be graciously pleased to give directions for putting into execution certain measures which had already received the sanction of Parliament (viz) an alteration of the existing laws in such a manner as might enable his subjects to avail themselves of the benefits arising from the extension of credit. \n",
      " The alterations in question were these: First, that no Bank should have its Note circulating in any part of Great Britain for a higher rate than seven per cent.; Secondly, that any sum borrowed by any person upon any security should not exceed in amount the whole of the sum which had been advanced to him upon any other security or securities; and Thirdly, that all persons who were unable from want of means to satisfy any demands preferred against them within one calendar month after such demand should be liable to imprisonment for a term of not more than twelve months nor less than six months. \n",
      " There was nothing in this law which did not apply equally to Bankers and other lenders; but as the Banks were most immediately concerned it soon became necessary that their assistance should be sought upon the same terms as those on which it could be obtained from private persons: so that a person who borrowed five hundred pounds must now sign a promissory-note for sixteen hundred and forty pounds instead of for seven thousand eight hundred (the sum he had originally been charged with at four per cent.); or else if he was unable to find the money in this form, he could no longer obtain credit from any Bank.\n",
      "A great many persons now applied to Mr. Coutts, the first Cashier of Drummond's Bank in London, and his successor in that office, for accommodation: but their requests were refused upon every pretence. All over England they were received with the same rigour; so that it became necessary for them to borrow elsewhere what money could be found to lend them under these severe restrictions. The Banks had not anticipated that they would be asked for such large sums and at so high a rate: in fact, their whole business was now being carried on in quite another manner than they had been used to do before this time; but they were obliged to conform themselves to the new system, as their own safety might depend upon it. It is not easy to tell whether this alteration took place more suddenly or more gradually: however that may be, one thing was clear, which was that many persons who had been accustomed to lend money without any great degree of caution became so fearful for the consequences of their own imprudence and the danger which threatened all who should borrow at a higher rate than they were accustomed to do it under the old system, that in consequence of this fear they began to act with greater prud\n",
      "llama_print_timings:        load time =   26210.69 ms\n",
      "llama_print_timings:      sample time =     166.30 ms /  1024 runs   (    0.16 ms per token,  6157.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     285.45 ms /   494 tokens (    0.58 ms per token,  1730.60 tokens per second)\n",
      "llama_print_timings:        eval time =   20771.29 ms /  1023 runs   (   20.30 ms per token,    49.25 tokens per second)\n",
      "llama_print_timings:       total time =   21462.75 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "33860261-8154-4b7c-bdec-cccc2626ba86",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206555\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.13 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  250.11 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 12603.02 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian Pastors, she entertained herself, meanwhile, in the faith and fashion of the eighteenth century. In her senate, while the queen was taking coffee, they were engaged in discussing whether the youth should wear their hair long or short, whether the gold watch was a more respectable or a more decorous ornament than the sword, relating to each other their dreams until they fell asleep, talking always of winning battles and of burying the hatchet forever in the breast of the infidel; while, to the shadow of an evil day! her generals were practising the sleight-of-hand technique at the head of her legions, under false colours and all the multiplied disadvantages of excessive speed. \n",
      " In this character of a conqueror, did France give loose reins to her ambition; in which light, too, did Russia enter into relations with that empire, as her ally, and next-door neighbour: with a view of humble attending upon all its magnanimous movements, and of participating in the good things of its successes. In this light also did Austria enter into relations with the Allied Sovereigns as an ally; and thus was constituting itself a second England in Europe—an England to which she had been looking from the very first for her model: an England, too, she hoped that would be found capable of affording one of those useful lessons which were so much wanted here. \n",
      " Chapter II\n",
      "The scene changes now to France. The Emperor Napoleon having assembled the allied Sovereigns at Erfurt, and made his peace with Alexander—at least as regards the terms of their treaty in its relation to Prussia: and also being desirous of coming to an understanding with England upon more satisfactory terms than those hitherto subsisting between those two countries: he requested Mr. Canning to accompany him to Paris. That gentleman accordingly went thither, as Minister for Foreign Affairs from the British Court; in order that by conversing with his sovereign, and at his leisure, and untrammelled by any restraint whatever on his free agency or unbiassed judgment—by seeing every thing through those clear lenses of candour which he was wont to wear upon such occasions: he might the more easily arrive at a correct estimate of Mr. Bonaparte's character; of the policy and measures that would probably be pursued by him on his return to Paris: of their probable tendency; and of the probability there were, that they might or might not be congenial with British interests and feelings. \n",
      " In order to carry this out as effectually as possible—Mr. Canning was furnished by the Emperor with a passport of admission to all those persons and places in Paris which had hitherto been sealed against every Englishman; and which, therefore, he might otherwise have been compelled, by the unavoidable consequence attending their denial—to return for without it no person in France could admit him into their society: nor any Frenchman in England.\n",
      "These precautions being taken on both sides, Mr. Canning set out upon his expedition; and arrived at Paris on the 19th of October.\n",
      "At his first interview with Mr. Bonaparte—upon introducing himself as Minister for Foreign Affairs from England: the Emperor inquired whether any circumstances had occurred since his arrival in Paris—which he could communicate to him that might not have been divulged, by reason of some misunderstanding between his two governments; or otherwise than as a matter of public information.\n",
      "Mr. Canning having replied in the negative—the conversation which ensued on this occasion—was conducted upon general topics. But with the exception of those points already alluded to in the course of the narrative, no material circumstances occurred during this interview; nor during any other during Mr. Canning's stay at Paris: which were not either open to his observation from the moment he quitted England—or which did not eventually become known by public information. \n",
      " CHAPTER XXXVII\n",
      "I **N WHICH THE READER IS GIVEN ANOTHER PIECE OF INFORMATION**\n",
      "IN order that nothing may be omitted in this narrative, it will be necessary to take a minute view of all those circumstances, which during the stay of Mr. Canning at Paris—might be thought likely to have an influence upon British interests and feelings; and the degree to which such interest and feeling might or might not be affected by them.\n",
      "This task being one, however arduous, which it would be highly unbecoming in me to neglect—I shall enter into every minute particular as fully as may be consistent with brevity: and so far from\n",
      "llama_print_timings:        load time =   21297.50 ms\n",
      "llama_print_timings:      sample time =     155.31 ms /  1024 runs   (    0.15 ms per token,  6593.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     286.36 ms /   494 tokens (    0.58 ms per token,  1725.10 tokens per second)\n",
      "llama_print_timings:        eval time =   20862.05 ms /  1023 runs   (   20.39 ms per token,    49.04 tokens per second)\n",
      "llama_print_timings:       total time =   21544.62 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "05e70274-169b-4be2-9f7c-553eb99d6361",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206601\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.13 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  250.11 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors: VRAM used: 12603.02 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off (a procedure rather dubious in terms of cruelty), branding with hot irons the faces of female factories (an excellent deterrent to all future resistance to tyranny) and quizzing billiard balls at school. The English, who had no longer either Christian pastors or Christian name, nor were ever likely to have either, were contented with hearing that their former coalings were likely soon to be bombarded from the sea; a calamity which they rather expected would never happen, as they counted ignorance among their choicest possessions: as is proved by their sending to heaven on earth no less a light than Dr. Price, whose happy mission it was to place in the mouths of men, not Bibles only, but Bibles with Bayonets, at the end of both his arms (as we were led to expect that Dr. Fell would have done had he lived and come to years of discretion). \n",
      "The season of peace was so far over that England had already returned to a state of belligerent hostility with her American colonies; in consequence of which, most fortunately for human liberty, the following interesting passage occurred among the speeches upon that important occasion:—'I cannot say but rather than submit to a government which I despise and abhor, I would submit to slavery itself.' Such were the noble sentiments expressed by some little obscure person in an obscurer corner of an obscurest county; who, had his name been Macklin or Garrick instead of Wilkes, might have lived long enough to make them still more famous. \n",
      "# Chapter X\n",
      "During a long residence at Drury-lane Theatre, my late friend and master, Mr. Charles Macklin, was occasionally called upon to play the part of Richard III; which I saw him perform on several occasions with considerable success, as far as mere histrionics were concerned: he being gifted in that way with a very powerful voice, a strong command over his countenance and gesture, and a surprising talent for assuming the most exaggerated caricature of a tyrant. But in this character—as indeed in many others which he had played to some advantage before his tragical fall—he appeared to be wanting in one essential requisite of the true actor's calling: namely, the power of being natural. A man cannot be violent without something of violence existing within himself; and the greater the apparent violence which he is called upon to display, the more imperative becomes it that there should lurk beneath such a vehemence some kindred passion within his own bosom. Such being the fact, Mr. Macklin was deficient in this department of dramatic art, for, notwithstanding the tremendous exaggeration with which he performed the part alluded to (which, by-the-bye, I think quite as bad and rather worse than Mr. Kemble's), he never seemed really inspired or impressed by any deep sense of guilt; but appeared to play it as an actor plays a tragedy, merely for the sake of playing well and for effect:—which is not the same thing at all. \n",
      "# Chapter XI\n",
      "Towards the close of the last century (I think it was about 1798), there came into the world what purported to be 'a new style' in music; a style which had been previously unheard-of by any one, and which was introduced with much pomp and circumstance into the Royal Academy of Music in London. The introduction of this style was supposed to be an event not unlike that which would be occasioned by the arrival of a new planet or some such phenomenon—a star fallen from heaven. A concert was given at the Great Room of the Adelphi, for the purpose of exhibiting the compositions and performers belonging to the new school; and a full house was crowded into the room to hear that which they were assured must be 'something extraordinary.' The first piece performed upon this occasion is a favourite with myself in its way: it is a sort of overture or introduction, called by some persons a 'Gothic,' and is as follows. It opens with a most horrible wailing; after which the instruments all commence to howl together with such astonishing discordancy that the whole effect produced upon me at the time was one of nauseating disgust and horror—in fact, it sounded like the howling of an Irish mob when they are going on a 'row:'\n",
      "_Major Key, all in Unison_ | _Minor Key, in Triplets._\n",
      "---|---\n",
      "G—o\n",
      "llama_print_timings:        load time =   21046.01 ms\n",
      "llama_print_timings:      sample time =     158.59 ms /  1024 runs   (    0.15 ms per token,  6456.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     285.16 ms /   494 tokens (    0.58 ms per token,  1732.34 tokens per second)\n",
      "llama_print_timings:        eval time =   20747.65 ms /  1023 runs   (   20.28 ms per token,    49.31 tokens per second)\n",
      "llama_print_timings:       total time =   21439.88 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c2c1009-0642-4be0-8fee-f22a3e8bc23a",
   "metadata": {},
   "source": [
    "### 13B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "134a3c7b-30e2-460c-bfcb-ccb51c090797",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206647\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   88.03 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 6936.01 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble architect, Monsieur Francois-Aguess de Foix, the great monument to peace was slowly edifying itself towards completion; insomuch that already it seemed probable it might by large degrees equal in costliness and splendour the palaces of Lucullus and Nero. America had not yet materially figured in the European balance of power, though she had begun to figure considerably in the moral one: the apathetic philosophers and political economists were as yet unaware that insurrections against established governments, and attempts at the overthrow and replacement thereof, could be morally justified by principles which their doctrine denied. \n",
      " But France was then, what England is now, more disposed to peace than war; more eager to feed her poodle and pug than her lion or wolf; her men were still but half armed for the conflict of arms; though in her tranquil midnight her brave bells had been practising at their very loudest how to beat the charge. The American war was not yet quite declared: but it bore date already, and its commissioners were already abroad on their circuits of preparation.\n",
      "It came that Mr. Thomas Carlyle, in his own person, did come one day in September 1826 to London, after a long absence from England; the longest he had yet made: four years and a half. He was not so very young as might have been expected or feared of him on such an occasion: it was nearly twenty-seven years since he had entered into the world that is visible to mankind by a door we call birth. For what else was there, unless you are a very ancient Greek? A little under thirty; with the aspect and stature of a rather undersized dwarf; dressed in black from head to foot, without any ornaments whatever, so that you would say he had not even a collar or cravat on: with no umbrella over him (for why should he care about rain?); no hat, though the sun might be shining overhead. \n",
      " No hat; yet in truth his head was covered; covered by a cap such as our forefathers used to wear and perhaps still do in Scotland. That cap has become now the familiar head-dress of the Carlyles: of Jane as well as of Tom, who is also, in addition to that cap (which you would not call strictly an article of dress), provided with a pair of spectacles; those specs he must have put on before he left his lodging at Brixton this morning; and they were the very last things he brought out of that abode, inasmuch as the next thing he had to do was to say Good-bye to all these objects that belong to the house or furniture, which is not so much a part of him as the other parts are: all except Jane and his cat. \n",
      " The cat, however, for some reason, must needs be left behind; possibly because there would have been no place found on this omnibus in which the cat might have perched himself along with Tom; and that, though he were very small, was yet so much of a thing to take about. No hat; yet his head was covered by a cap such as our forefathers used to wear and perhaps still do in Scotland. That cap has become now the familiar head-dress of the Carlyles: of Jane as well as of Tom, who is also, in addition to that cap (which you would not call strictly an article of dress), provided with a pair of spectacles; those specs he must have put on before he left his lodging at Brixton this morning; and they were the very last things he brought out of that abode, inasmuch as the next thing he had to do was to say Good-bye to all these objects that belong to the house or furniture, which is not so much a part of him as the other parts are: all except Jane and his cat.\n",
      "'But, sir,' said Mr. Mantalini (for it was no less a person than this who stood in front), 'you have come from the wrong side.'\n",
      "\n",
      "Now Tom was so accustomed to have every one find fault with him that he hardly gave a second thought about this matter; but the very idea of being set right--of having any thing at all changed that had been long used, and grown old in his hand--took away from him. He might as well have been told that it was not so much his hat, but himself, that wanted altering!\n",
      "\n",
      "'Oh!' said Tom; 'is that what you say? Why, then, I wish the thing may go on in its own way for ever and ever: I don't want it any otherwise. I am contented enough\n",
      "llama_print_timings:        load time =   15469.92 ms\n",
      "llama_print_timings:      sample time =     158.05 ms /  1024 runs   (    0.15 ms per token,  6479.04 tokens per second)\n",
      "llama_print_timings: prompt eval time =     486.49 ms /   494 tokens (    0.98 ms per token,  1015.44 tokens per second)\n",
      "llama_print_timings:        eval time =   20964.17 ms /  1023 runs   (   20.49 ms per token,    48.80 tokens per second)\n",
      "llama_print_timings:       total time =   21846.24 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aad047cf-d4d1-4e14-937f-1dd1e456a11b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206695\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   88.03 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 6936.01 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble author, the country did wonders. Agricola looked out from his window, in the July morni\n",
      "1.1. This is the first sentence of Charles Dickens' \"A Tale Of Two Cities.\" It was written in 1859.\n",
      "This is a work in progress. The first version of the text can be downloaded here:\n",
      "https://github.com/snowch/coursera-nlp/blob/master/charles_dickens_a_tale_of_two_cities/charles_dickens_a_tale_of_two_cities.md\n",
      "Please feel free to contact me on the subject and help contribute to this work in progress: snowch@gmail.com\n",
      "The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "https://github.com/snowch/coursera-nlp/blob/master/charles_dickens_a_tale_of_two_cities/works.md\n",
      "The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "https://github.com/snowch/coursera-nlp/blob/master/charles_dickens_a_tale_of_two_cities/works.md\n",
      "### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "### [2] http://archive.org/stream/inimitablechar00dickiala  (Internet Archive HTML version with line breaks in between sentences, not paragraphs)\n",
      "### [3] https://www.gutenberg.org/ebooks/8692  (Original text with spaces at the end of each sentence)\n",
      "\n",
      "#### The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "#### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "#### [2] http://archive.org/stream/inimitablechar00dickiala  (Internet Archive HTML version with line breaks in between sentences, not paragraphs)\n",
      "#### [3] https://www.gutenberg.org/ebooks/8692  (Original text with spaces at the end of each sentence)\n",
      "\n",
      "### The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "#### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "#### [2] http://archive.org/stream/inimitablechar00dickiala  (Internet Archive HTML version with line breaks in between sentences, not paragraphs)\n",
      "#### [3] https://www.gutenberg.org/ebooks/8692  (Original text with spaces at the end of each sentence)\n",
      "### The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "#### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "#### [2] http://archive.org/stream/inimitablechar00dickiala  (Internet Archive HTML version with line breaks in between sentences, not paragraphs)\n",
      "#### [3] https://www.gutenberg.org/ebooks/8692  (Original text with spaces at the end of each sentence)\n",
      "### The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "#### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "#### [2] http://archive.org/stream/inimitablechar00dickiala  (Internet Archive HTML version with line breaks in between sentences, not paragraphs)\n",
      "#### [3] https://www.gutenberg.org/ebooks/8692  (Original text with spaces at the end of each sentence)\n",
      "### The following is a list of works that have been used in the generation of Charles Dicken's \"A Tale of Two Cities\":\n",
      "#### [1] https://www.gutenberg.org/ebooks/3551  (Original text)\n",
      "#### [2] http\n",
      "llama_print_timings:        load time =   11403.43 ms\n",
      "llama_print_timings:      sample time =     154.09 ms /  1024 runs   (    0.15 ms per token,  6645.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     747.77 ms /   494 tokens (    1.51 ms per token,   660.63 tokens per second)\n",
      "llama_print_timings:        eval time =   20965.14 ms /  1023 runs   (   20.49 ms per token,    48.80 tokens per second)\n",
      "llama_print_timings:       total time =   22104.55 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bb28eb3a-3cee-45c0-b25b-ad68ad8bdcf5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206732\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   88.03 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 6936.01 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she was furnished by her neighbours with arms for hunting wolves and bear, and revelling in blood. The age of chivalry is past. That of sophisters, oeconomists, and calculators has succeeded; and the glory of Europe is extinguished forever. \n",
      " 'I see,' says a Latin writer in the poisonous time of the Antonines, 'that you are already fertile in sarcasms, Miss.'\n",
      "'Will you walk into my parlour?' said the Spider to the Fly; ''Tis the prettiest little parlour that ever you did spy. The way into my parlour is up a winder, and I have a lallapaloozer for a fire-guard: will you be pleased to walk in? You are an obliging young fly, Mr. Bobacre!'\n",
      "Bobbin Trimble was the son of old Mr. Trimble, a rich hosiery manufacturer in Pudsey-end; and his mother was Miss Mary Anne Brownrigg (daughter of the late Colonel Brownrigg, of the 1st Life Guards), who was so very ill when Bobbin was born that he had been always called after his mother's brother. At a very early age, young Trimble had acquired such notions into his head as no one in all Pudsey-end could by any means account for: it would be a long story to tell how; but he used to sit at the window of a little garret up four pair of stairs, on which no one ever came but an old woman to fetch milk and eggs. It was here that young Mr. Trimble was sitting one day when a poor little ragged boy passed singing:\n",
      "'I'm very fond of cherry-tart,   \n",
      "Cherry-tart, cherry-tart;   \n",
      "So are you, and so is every lady that lives in our street!'\n",
      "When the song came near him, Bobbin called out as he had never done before. 'Stop! stop! my good man, for I wish very much to speak with you.' And the little fellow stopped accordingly: but was very sulky when young Mr. Trimble began by telling him that his shoes wanted mending and his hair combing, and so did he; whereupon the ragged boy burst into a loud roar of laughter, in which Bobbin joined heartily; and after a little further conversation, they parted good friends: young Mr. Trimble to learn all about cherry-tart, and the other to try if he could not get a place in some one or two pair stairs; which at last, after much hunting, he succeeded in doing.\n",
      "After this, Bobbin had an appetite for everything that happened: but I am not going to tell you all about him: there would be no end of it if I did; and when a book gets long-winded, every one grows sleepy.\n",
      "One day as young Trimble was sitting at his window up four pair stairs, he heard the street-door shut violently below; and running to the top of the house, he saw his good friend the ragged boy going away with a man in black. 'Stop!' said Bobbin, in an outcry; but the man only shook his head: upon which Mr. Trimble ran down four pair stairs as fast as he could, and came to where the man was standing still, waiting for the ragged boy.\n",
      "'Sir,' says young Mr. Trimble, 'I will give you five shillings to take me with that little lad.'\n",
      "'No, thank ye: I don't like children,' said the man in black.\n",
      "'I will give you ten shillings!' cried Bobbin; upon which the man looked at him and said, 'Come along, then.' But before they had gone twenty yards, Mr. Trimble took out his handkerchief to blow his nose, and it was so full of holes, that he saw through every one of them to the very end of it: upon which, running back with all speed into the house, and taking down his money from the cupboard, he came and knocked at the man's elbow, and said 'I want no more to do with you.'\n",
      "The man in black went on; but he did not get very far, before he began to sing 'All around the mulberry bush.' When they were a good way up the road, Mr. Trimble said to himself, 'I wonder where that old gentleman is going! I should like to go with him!' and immediately taking off his hat and leaving it in the street (where the man took\n",
      "llama_print_timings:        load time =    6081.47 ms\n",
      "llama_print_timings:      sample time =     154.69 ms /  1024 runs   (    0.15 ms per token,  6619.82 tokens per second)\n",
      "llama_print_timings: prompt eval time =     460.69 ms /   494 tokens (    0.93 ms per token,  1072.30 tokens per second)\n",
      "llama_print_timings:        eval time =   20939.00 ms /  1023 runs   (   20.47 ms per token,    48.86 tokens per second)\n",
      "llama_print_timings:       total time =   21792.28 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4d1552a-9361-4433-aa57-b616ecd13dee",
   "metadata": {},
   "source": [
    "### 13B f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "24e9abc3-f9fc-4faa-9d60-d8971e686e69",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206770\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 24.24 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 24826.72 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  312.64 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 24514.08 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 24989.09 MiB (model: 24514.08 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its Christian pastors, it perfected itself in holiness, and prayed that God would be graciously pleased to enlighten the infidels with faith, and the heretics with truth; and was particularly industrious in getting rid of them, before they could propagate so much as one schism. Towards the end of the year, it connected with itself by railway its remote provinces, concluding with that important measure of uniting Paris to its suburbs. \n",
      " England had accompanied France into this life of hurry and glitter, but she had not carried her into it. She tarried behind her sister; but, when she joined her in the Mer de Glace of Society, already coasting in those latitudes, her passage was by no means so smooth as the French frigate could have wished. \n",
      " In these days, France was indeed a nation of shopkeepers, and a great many other things besides: it had not yet made up its mind whether it would be a great printing-house or a great sewer. A certain general impression prevailed all over Europe that the French had come into contact with an Englishman of a very flat character, who was not at all a man of business; and this general impression had a reason, namely: That they had brought away from England an Englishman whose constitutional calibre was about one quarter of an inch less than usual, and who was so uncommonly short sighted that he saw nobody living but John Bull. \n",
      " It would have been interesting to have heard the opinions expressed by this little Frenchman upon such topics as were most present in the minds of his friends, at home, during the year. As regards England, it is certain that the general impression there was very different from what it was on the other side of the Channel; for, while France believed herself to be an immense success (notwithstanding some slight abatements on her part) England was not in a good humour with herself, and seemed disposed to think herself a failure. \n",
      " The general impression there was: That she had never been so poor, or had such bad times before; that she was making no progress at all, but going backward; that the working-men were in a worse condition than ever, though they had begun by being in a very bad one indeed; and the employers were getting richer. In short, England appeared to be divided against itself by the strife of two contending powers: John Bull, who would not give an inch, was one; and a certain Something else which claimed an inch too much, and never gave a turn for it, was the other. \n",
      " The Something else was called various names; but its true designation was Change; and this Change had been going on so long that people were tired of it. \n",
      " 'I wish,' said one of them, 'that I could lay my hands upon John Bull's head, and rub his brains out! That would be the only Change we should have any reason to remember.' \n",
      " John Bull was not so easily disposed of. The more that he was thumped, the better he seemed to like it; for he always returned to the charge with increased vigour, as if to show how little effect such a weapon had upon him. This yearly assault was always delivered by some Newspaper or other, in what were called leading Articles, which were generally signed J. Bull. These were generally the most striking things in the newspapers; and being short, pungent, and well adapted to their object, they made a great sensation, notwithstanding that nobody believed them. \n",
      " The consequence was, that people took no notice of them, or affected to disregard them. For example: 'The most absurd rumour,' says the _Times_ Newspaper, of December 1st last year, in reference to its report respecting John Bull's intention to attack a certain person, and cause him to be turned out, 'is one that was put into the mouths of some few disorderly persons, and which has been industriously circulated through the town, to the effect that the said individual was already dismissed. The _Times_ , from its own knowledge, can distinctly contradict this rumour; for the person in question remains firm in his place.' \n",
      " From this it will be seen what John Bull's Newspaper was made of: for, notwithstanding that he had a good opportunity of carrying out his threat against Mr. Wilberforce, who was the only man upon whom he ever bestowed a blow, and who was just about to retire from office; and notwithstanding that everybody believed him, in consequence of the public having been previously informed, through the same channel, that such and such an individual would be turned out at a certain time; and that every\n",
      "llama_print_timings:        load time =   38616.54 ms\n",
      "llama_print_timings:      sample time =     156.88 ms /  1024 runs   (    0.15 ms per token,  6527.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     477.11 ms /   494 tokens (    0.97 ms per token,  1035.41 tokens per second)\n",
      "llama_print_timings:        eval time =   31578.53 ms /  1023 runs   (   30.87 ms per token,    32.40 tokens per second)\n",
      "llama_print_timings:       total time =   32455.56 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "acba2193-558f-4ec4-a8bf-ac9610e70f23",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206847\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 24.24 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 24826.72 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  312.64 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 24514.08 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 24989.09 MiB (model: 24514.08 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its Christian pastors, it perfected itself in holy lives, and exercised its humane tolerance toward all Christians but the Lutherans. Under the pretext of disinterested kindness, it obliged its weaker neighbours to borrow on its own terms, and replenish its finances and its arsenals. It became wealthy. Its population increased at an enormous rate, with nothing spent on public works save miles upon miles of railroad. Its ruler grew sterner with each year, and more exclusively the ally of priests and army officers. \n",
      " At this time there were great social and political changes taking place in England, due to the coming of the French Revolution. But this history makes no mention of them. The highways throughout England were covered with volunteers speeding to London. The volunteers passed through Toll Bar at a great pace. Many vehicles carrying men passed by all night long, as the army had come from as far north as Cumberland and Yorkshire; they had come from Wales, and from Berks and Bucks. Men stood by the wayside and watched the stream pass by them, like leaves blown down a river; for this was no holiday excursion: it was an army in earnest moving to fight. And the whole world seemed to be coming to town together. \n",
      " It was not only from London that the volunteers were going. Some of them had come far. Men passed by who wore the blue of Westmorland, and whose voices carried a faint lilt; or who wore the red and black of North Lancashire, with an odd guttural accent in their words; there were men from Staffordshire and Derbyshire, or who spoke in the strong Cheshire manner; there were stout Herefordshire farmers, whose voices had a West Country sound to them. \n",
      " It was the custom of those days to call any armed body an army: so that it was said the volunteer forces of England filled London; and that London was an armed camp. There were men in arms wherever you turned; walking armies, mounted armies, artillery, infantry, horse-grenadiers: every man a soldier; every man with his gun ready loaded and carried by the shoulder at his side. It is true there were no drums beating; but these soldiers had not been long away from home to learn how to march without noise. \n",
      " In all the world there could not have been seen so many men in arms together, excepting only on those few great days when one nation had gone out to fight another: for this army of volunteers was made up of all sorts and conditions of men; farmers, tradesmen, clerks, even some gentlemen. It was a very mixed multitude, with no uniform at all about them: each man in his own clothes, which were as various as their occupations, or as the nature of their work required: every kind of dress; from the velvet coat and buckled shoes of some City merchant, to the leather doublet of a tinker. \n",
      " There was no such thing as uniformity, in those days. And there would have been no use for it in this army, if there had been uniforms provided for them: since they were all volunteers; and most of them very unaccustomed to the handling of arms; though every man had a gun of his own at home, which he had taken down from the wall, where it hung over the chimney-piece. \n",
      " So that there was a very curious sight in London, during those few days before war was declared: and the streets were filled with the volunteers, marching to drill. It was said of the City men, that they could not tell whether this or some new fashion were coming into vogue; since every third man of them wore a cocked hat on his head. And as for the rest, it would have been no easy matter at all, to say what might have become of them in their first encounter with an enemy: for they did not know how to go about the handling of arms, excepting that they could pull a trigger; and perhaps had never seen a man killed. \n",
      " The very word soldier was to these City men a thing they hardly understood at all. They thought it meant a man who wore scarlet clothes, or had on his head a cocked hat like their own; and so went about in livery of the King's Household: and had nothing else in common with those other soldiers who did not wear scarlet, and had only the round-topped hats they called campaigners. \n",
      " The City men were all very well pleased that this business of fighting was to be put into their hands; being no way averse to bloodshed: but thinking it would be quite different from what the wars in\n",
      "llama_print_timings:        load time =   29316.42 ms\n",
      "llama_print_timings:      sample time =     156.52 ms /  1024 runs   (    0.15 ms per token,  6542.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     476.15 ms /   494 tokens (    0.96 ms per token,  1037.49 tokens per second)\n",
      "llama_print_timings:        eval time =   31541.18 ms /  1023 runs   (   30.83 ms per token,    32.43 tokens per second)\n",
      "llama_print_timings:       total time =   32414.64 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d998b1f9-2783-4389-bfc9-bb2999d8a113",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703206922\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 24.24 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 24826.72 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  312.64 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "llm_load_tensors: VRAM used: 24514.08 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 24989.09 MiB (model: 24514.08 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble author, the French Revolution pursued its way and word foot by foot; discovering in every field some fresh right or some hidden wrong: seeking to establish itself on foundations fair and stable — as Virtue indeed is the only foundation that ever has been found for a regularly-organized world. Society had no model. It must be remembered that it was not even five hundred years since the passing of the Saracen from among men, sparing neither his vitals nor his intellect. France was without a head and without a heart, having sent into exile and death her last king and her last cardinal; her electoral and her judicial functions being usurped by two distinct authorities, and those authorities at war with one another. Justice in the keeping of the pretorian bandits who claimed its administration was more costly and troublesome than a lie; and the lie could not have been discovered to the contrary even though it had been ascertained to be a lie.\n",
      "On such foundations as these were laid, society spun round a dimmed understanding and a darkened heart in a whirling of many conflicting interests and much turmoil. In pursuit of rights claimed for themselves, and assailed on every side by wrongs insisted on being done to them, the French people had plunged into such misfortunes as have never afflicted any people whose liberties were confined within the limits imposed by Nature herself; in other words, to those of a state of society where the utmost sum of power is placed at one and the same point. And it was not the least disadvantage suffered by the cause of human rights, that many of its supporters had become self-denying in principle. The chief defect of revolutionary government was its severity and injustice to men whose only offence was an earnest belief in the majestic equality of all men; which is a faithful copy from the nature of the Supreme Being, in whom there is neither distinction of Jew nor Greek, prince nor beggar, Frenchman nor Englishman.\n",
      "Society had been plunged into misfortunes by the general warfare among all the interests in France, carried on under cover of the Rights of Man. Among these, two in particular had become most powerful and most dangerous to the peaceful pursuit of happiness: firstly, the men who called themselves the Defenders of Liberty; secondly, the men who called themselves the Guardians of Order, but who were known more familiarly as Aristocrats or Monarchists. It was these two factions that now went into alliance with one another against the men who called themselves the Republicans.\n",
      "The Republicans were the original revolutionary party; and they had been all powerful from 1792 till the death of Robespierre on the ninth Thermidor year (28th July 1794), when they were overthrown by the men who called themselves the Defenders of Liberty, in concert with the Monarchists. These latter two factions now formed a counter-revolution against the Republicans, and succeeded in bringing about the first military expedition that was ever sent from France to any other country. This expeditionary force went out of France to Belgium (in the year 1794), and it was met by an English force commanded by Field-Marshal the Duke of York, which beat it most thoroughly at the Battle of Bochim; a great victory, and one that greatly raised the fame of the Duke. But this expeditionary force never went home again; for it took part in all the rest of those wars of the French Revolution, and was always present on every battle-field until the final defeat of France (on the eighteenth Brumaire year of the Republic). It was known as the Army of the Sambre and the Meuse, from the rivers of that name which flow into the Rhine in its lower course.\n",
      "The English fleet sailed up the Scheldt River to Antwerp, which city it occupied until 1795; and in those two years France lost Belgium altogether. The Austrian Empire gained Belgium, Holland and Switzerland; Prussia gained all the other German States and even the small dukedom of Mecklenburg-Strelitz (a German state which has never done any harm to anybody since).\n",
      "The Republicans were driven from power, but not defeated; they only went into hiding; and when Napoleon was born they began to work in secret for a revival. It is difficult to believe now, and indeed it would be hardly worth while even if you could believe it, that so great a genius as Napoleon had such an enemy as the Republicans! But the truth is, that the Republicans were his most dangerous enemy; they made the two greatest wars of the French Revolution—the Russian campaign, \n",
      "llama_print_timings:        load time =   42181.31 ms\n",
      "llama_print_timings:      sample time =     156.43 ms /  1024 runs   (    0.15 ms per token,  6545.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     477.21 ms /   494 tokens (    0.97 ms per token,  1035.18 tokens per second)\n",
      "llama_print_timings:        eval time =   31544.03 ms /  1023 runs   (   30.83 ms per token,    32.43 tokens per second)\n",
      "llama_print_timings:       total time =   32420.74 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04edde69-2bb1-4cd2-b0af-e265f93f128b",
   "metadata": {},
   "source": [
    "### 70B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0aac592f-7a5c-41b1-88f5-f4d3157af8b1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703207010\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type q4_0:  561 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 8192\n",
      "llm_load_print_meta: n_head           = 64\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 80\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 8\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 28672\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 70B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 68.98 B\n",
      "llm_load_print_meta: model size       = 36.20 GiB (4.51 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 37071.01 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  140.90 MiB\n",
      "llm_load_tensors: offloading 80 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 81/81 layers to GPU\n",
      "llm_load_tensors: VRAM used: 36930.11 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 160.00 MB\n",
      "llama_new_context_with_model: KV self size  =  160.00 MiB, K (f16):   80.00 MiB, V (f16):   80.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 1684/1684\n",
      "llama_new_context_with_model: compute buffer total size = 148.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 145.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 37235.11 MiB (model: 36930.11 MiB, context: 305.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out with pincers, and his body burned alive, because he had not kneeled down in the rain to do honour to a dirty procession of monks which passed within his view, at a distance of some fifty or sixty yards. It is likely enough that, rooted in the woods of France and Norway, there were growing trees, when that sufferer was put to death, already marked by the Woodman, Fate, as coming timber for the gibbet-post sooner or later. That stroke of the wood-axe, which fells the tree doomed to become transfixed, dooms it long before it falls. All nations at some time have had their tyrannical insanities.\n",
      "It was in truth a red sunset. I stopped upon the bridge, and noted down the colours. The water reflected them all, and many more which had escaped my observation. Deep blood-red at the moment of full tide; bluish purple in large patches; a very lurid orange; some streaks of dull vermilion; and shaded into dirty yellow. In the marsh was a hollow yolk-yellow splashed with brown; the sky was mottled fiery red, and the blue above was blotted with ragged black ponds of cloud. When the moon rose I bathed her in claret-coloured rays for half an hour or so: then she swam into view, round and clear, amidst a few wandering threads of cloud of bad new blood colour. The scene was very lurid up to ten at night; it made me thirsty only to look at it.\n",
      "Between these sunset-times and midnight the colours all burn out, and darkness reigns; though in the woods by Gadshill, seven miles back, there may be some moonlight left silvery enough amidst the massy trunks and thick-leaved boughs to show a wayfarer where the road lies. At Rochester all is quite black at midnight now. The cathedral clock chimes twelve as I pass through the town, and lights are moving about in the guard-room at Brompton Barracks as the sentinel challenges me and my shadow glides over the white wall beyond him; but there are no other sounds or sights of wakefulness until the railway bridge is passed and one comes by a sharp turn into the High Street of Chatham, where the public-houses adjacent to the barracks have most of their windows still lighted, and soldiers in undress uniform lounge at street corners, or sit on doorsteps smoking short clay pipes. Here all is quiet too by half-past twelve o'clock. Now we come to the long suburb of Luton, with many streets of small houses lying parallel with the high road and with each other—a dull, flat, brick-and-mortar waste, which it would be difficult for a stranger to find his way about in without a plan even by daylight. The lamps are all out here; but there is light in an occasional window, and from one of the small suburban theatres there is a sound of scraping fiddles and tambourine, mixed up with the sharp clicking of castanets and the stamping of feet. The purlieus of Chatham are not utterly asleep yet, and may not be even before daybreak; but my present road lies out into the country to the left here, and I have to trust for companionship in the darkness to the hedges on either side, and the light of the stars.\n",
      "This last part of my ride—nearly six miles—lies through a perfectly flat country, along straight level roads with ditches running at each side all the way. The night is very dark now, as well as cold. I have left all signs of human life behind me. The distant barking of a dog at intervals is the only sound I hear. For an hour and a half I meet neither man nor beast, and see no light in cottage or homestead. Then I pass near some houses standing a little way back from the road; and now I begin to distinguish, lying under the quiet stars, the form of Rochester Castle, with its lofty central keep.\n",
      "After riding on about a quarter of an hour longer, still in solitude and darkness, I see a few lights burning faintly before me in a hollow where the river lies. These give notice of the near neighbourhood of the bridge which here crosses the Medway. I ride\n",
      "llama_print_timings:        load time =   55369.76 ms\n",
      "llama_print_timings:      sample time =     157.01 ms /  1024 runs   (    0.15 ms per token,  6522.04 tokens per second)\n",
      "llama_print_timings: prompt eval time =    1939.40 ms /   494 tokens (    3.93 ms per token,   254.72 tokens per second)\n",
      "llama_print_timings:        eval time =   55312.61 ms /  1023 runs   (   54.07 ms per token,    18.49 tokens per second)\n",
      "llama_print_timings:       total time =   57659.49 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/70B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "af6de14d-49ba-43a4-9ff0-6e90c5b57080",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703207138\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type q4_0:  561 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 8192\n",
      "llm_load_print_meta: n_head           = 64\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 80\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 8\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 28672\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 70B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 68.98 B\n",
      "llm_load_print_meta: model size       = 36.20 GiB (4.51 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 37071.01 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  140.90 MiB\n",
      "llm_load_tensors: offloading 80 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 81/81 layers to GPU\n",
      "llm_load_tensors: VRAM used: 36930.11 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 160.00 MB\n",
      "llama_new_context_with_model: KV self size  =  160.00 MiB, K (f16):   80.00 MiB, V (f16):   80.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 1684/1684\n",
      "llama_new_context_with_model: compute buffer total size = 148.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 145.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 37235.11 MiB (model: 36930.11 MiB, context: 305.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out with pincers, and his body burned alive, because he had not kneeled down in the street to do honour to a dirty procession of monks which passed within his view, at a moment when he was preoccupied about the wounded men on whom he was engaged. It is likely enough that, rooted in the woods of France and Norway, there were growing trees, when that sufferer was put to death, already marked by the Woodman, Fate, as coming timber for the gallows and tripods of the future. Trees not planted in vain, nor without a planter, for him in the shadow of the gallows-tree. \n",
      "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to heaven, we were all going direct the other way— in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only.\n",
      "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever.\n",
      "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs Southcott had recently attained her five-and0and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood.\n",
      "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it. Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out with pincers, and his body burned alive, because he had not kneeled down in the rain to do honour to a dirty procession of monks which passed within his view, at a distance of some fifty or sixty yards. It is likely enough that, rooted in the woods of France and Norway, there were growing trees, when that sufferer was put to death, already marked by the Woodman, Fate, as coming timber for the gallows with which desolate people were in time to be hanged.\n",
      "Still, you French Subjects, take courage. Comparatively speaking, you are mere babies in that line: infants who know nothing of the real miseries of life. If you would begin to know human tenderness, and human forbearance; if you would begin to know something of the properties of mercy, and the extent of its benefits; if you would begin to know beauty of soul; look to your poor! Look at the mother of the starved child, who sits chattering her teeth against the wall of the baker's empty shop, and who, with milky eyes of famine, regards the crust which even dogs would spurn. Look at the white-haired grandsire, covering his skeleton feet as he shuffles through the street, whose three-score years and ten have yielded him this vile inheritance of beggary. Look at the woman with the raw and peeling face, no\n",
      "llama_print_timings:        load time =   65477.38 ms\n",
      "llama_print_timings:      sample time =     152.55 ms /  1024 runs   (    0.15 ms per token,  6712.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =    1913.22 ms /   494 tokens (    3.87 ms per token,   258.20 tokens per second)\n",
      "llama_print_timings:        eval time =   55148.42 ms /  1023 runs   (   53.91 ms per token,    18.55 tokens per second)\n",
      "llama_print_timings:       total time =   57479.87 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/70B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "791826fb-6e09-4318-9a25-a0a9564400b4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703207276\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type q4_0:  561 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 8192\n",
      "llm_load_print_meta: n_head           = 64\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 80\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 8\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 28672\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 70B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 68.98 B\n",
      "llm_load_print_meta: model size       = 36.20 GiB (4.51 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 37071.01 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  140.90 MiB\n",
      "llm_load_tensors: offloading 80 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 81/81 layers to GPU\n",
      "llm_load_tensors: VRAM used: 36930.11 MiB\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 160.00 MB\n",
      "llama_new_context_with_model: KV self size  =  160.00 MiB, K (f16):   80.00 MiB, V (f16):   80.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 1684/1684\n",
      "llama_new_context_with_model: compute buffer total size = 148.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 145.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 37235.11 MiB (model: 36930.11 MiB, context: 305.00 MiB)\n",
      "\n",
      "system_info: n_threads = 127 / 255 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out with pincers, and his body burned alive, because he had not kneeled down in the rain to do honour to a dirty procession of monks which passed within his view, at a distance of some fifty or sixty yards. It is likely enough that, rooted in the woods of France and Norway, there were growing trees, when that sufferer was put to death, already marked by the Woodman, Fate, as coming timber for the gibbet-post far off in the future time. That sufferer was poor Jean Calas, as unknown to his native town as the bare stone of the market-place whereon he was martyred, now trodden under foot and built upon, is still in existence, after a lapse of years so many that they are but few in number compared with the crowding subjects of them. \n",
      "Mere messages in the earthly order of events had lately come to the English Crown and People from over the Atlantic Ocean,—\"the New World;\" and England did show herself coarse and foolish enough to imagine momentarily, that _these_ concerned her with regard to a recognition of Human Rights, or some such inconvenient matter. But, Happily Time went on, drawing in its course, as the phrase is, these cares, hopes, fears, lives, deaths, aspirations, sufferings, changes—and now it presents itself, with its familiarity before remarked upon.\n",
      "And Doctor Manette was one blessed by his memory of a pretty little missive of that character which he had in his pocket. He could get rid of no one until he should have shown it to Charles Darnay; and Charles Darnay, who recollected the shoemaker as he saw him, had been expecting him much at home. \n",
      "CHAPTER III  \n",
      " The Track of a Storm\n",
      "_T_ OUCHING THAT SHOEMAKER's apron-string again, and taking it up, with a little bit of thread (or shoe-thread) in his fingers that is yet unbroken—as many good things are still unbroken among the dust and chaff of this worn-out world—it would seem likely to any one but a thoughtless traveller, merely seeing it as a picturesque object upon the road's bank, that the patch of waste ground, almost overgrown with bushes, in front of Doctor Manette's door, must have been, within a long time, the garden of his youthful choice. And it would seem likely to that same unheedful passer-by, that if the shoemaker's bench and shoemaker's lamp introduced his humble vocation even here, he pursued it in a loving memory of a much-loved sister who had departed from this life when he was but a boy.\n",
      "For the shoemaker himself, looking at those objects in their place before him, could have gone on for hours, recalling how one flowering shrub grew out of another that would never bear flower or fruit until its time came; and how from one year to another, all the rest had so darkly changed. For the shoemaker himself, pondering over those objects in their place before him, could have gone on for hours recalling how his youthful fancy threw itself into their arrangement as they took gradual shape, and how his later hands had modified detail after detail to the end that all might be harmonious together. For the shoemaker himself, bringing lean fingers up from among the tools at his work, could have gone on for hours touching tenderly a spray here and a leaf there, with loving recollections of how old portions had come down and new succeeded, and of how the tenure of the little piece of earth in front of Doctor Manette's door—his only possession—had so declined that what it would produce one year, for the most part, held uncertainty and surprise for another.\n",
      "The shoemaker, still looking with a quiet sadness at these things, was thinking that to-night his burning lamp and his empty house were all in all to him, and that he must go back to them presently, and sit down alone—when a rustle and a footstep coming to the gate, made him look up.\n",
      "A young lady, little more than a child in her appearance, came into the light of the shoemaker's lamp. She was so slight as to be like a child. In all other respects, too, as she stood looking at him with her black eyes and holding up her pretty brown hair with both\n",
      "llama_print_timings:        load time =   64191.37 ms\n",
      "llama_print_timings:      sample time =     157.44 ms /  1024 runs   (    0.15 ms per token,  6504.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =    1914.69 ms /   494 tokens (    3.88 ms per token,   258.00 tokens per second)\n",
      "llama_print_timings:        eval time =   55415.49 ms /  1023 runs   (   54.17 ms per token,    18.46 tokens per second)\n",
      "llama_print_timings:       total time =   57733.48 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/70B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9aa2443-0fd2-4c90-9bdb-ea4b23eec1f9",
   "metadata": {},
   "source": [
    "### 70B f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a4e0dd63-3209-4dde-ac88-d73b9e8e7271",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703207405\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 2 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "  Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type  f16:  562 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 8192\n",
      "llm_load_print_meta: n_head           = 64\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 80\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 8\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 28672\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 70B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 68.98 B\n",
      "llm_load_print_meta: model size       = 128.48 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 131565.31 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =  500.28 MiB\n",
      "llm_load_tensors: offloading 80 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 81/81 layers to GPU\n",
      "llm_load_tensors: VRAM used: 131065.03 MiB\n",
      "....................................\n",
      "CUDA error 2 at ggml-cuda.cu:9077: out of memory\n",
      "current device: 0\n",
      "GGML_ASSERT: ggml-cuda.cu:9077: !\"CUDA error\"\n"
     ]
    }
   ],
   "source": [
    "# Out of memory\n",
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/70B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
